diff --git a/README.md b/README.md index aac0bad..92a8c49 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Built for geographic resilience with single-cluster operational simplicity: thre deployment/modules/ ├── ovh/account/ # cloud project, vRack, private network, CPs, workers, IPLB, DNS ├── netbird/cluster/ # per-env mesh: vRack route, mesh-gateway VIP + DNS, pod egress, policies +├── grafana/cluster/ # Grafana service account for the Rootly integration ├── netbox/cluster/ # IPAM registration of the ranges the other modules allocate ├── talos/cluster/ # machine secrets, CP + worker configs, bootstrap, ingress firewall └── kubernetes/helm/ # CoreDNS, Flux Operator + Instance, bootstrap-settings, env secrets diff --git a/deployment/modules/grafana/cluster/.terraform.lock.hcl b/deployment/modules/grafana/cluster/.terraform.lock.hcl new file mode 100644 index 0000000..3efebd8 --- /dev/null +++ b/deployment/modules/grafana/cluster/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/1password/onepassword" { + version = "3.3.1" + constraints = "~> 3.3" + hashes = [ + "h1:35PCpSNLVubReT1imwfC+FpIP5gQWx+rvG4njkXkZKM=", + "h1:GqvBYImDNYNPMi9o3kJTSIJDeEnNO7om46RWA0wjeso=", + "h1:Tg8bJ+ATy5pla6ZTn87lb3nIyiTJTgQHJdxOWQjFF9k=", + "h1:a8DHdeyXt8YaEngXGEWZC75W27OG742wtw5BmRnhrpI=", + "h1:bPFt8A+PWGwEj3wQ3D0GxyrTMmr1piNrwEuPJu7AZTo=", + "h1:nThx/0XctUKlAa6aAsSEHZvx/mD1tSxCE6pacSnotxU=", + "h1:syh+iS5VPBQTLszL503BOi5rLpL1wGl1IjyYqgpKYIg=", + "zh:02d93a7f520ec69ad8944a68dcbf512e2f9920a6696628b8d05e6ad408309f35", + "zh:0f91a902da84470af95f0da4dc21127b84e23c856a431ff9ecfe45d9c6775ef0", + "zh:161bc55c466214a5d425ba85753d74ed5078212db965f726e6650d2e1524d633", + "zh:3de4a9f212e1046016a3ace8816e2cb15cfb7b9579161e468a08b9034d6a5f51", + "zh:730105346065ea3d2bd6acc6f5fe36f7b8a2b54c513d20a46bcd51d656e82bb4", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:c29f6025d099bc8f1f96e7bb4cd66c5d07209b4141d2fd7720228cf20c9c8efd", + "zh:d8ea431d396986ca6baf033fa9aaccb73d6a9f9b7d42bea7af8dc73b9ef20297", + ] +} + +provider "registry.opentofu.org/grafana/grafana" { + version = "4.46.0" + constraints = "~> 4.46" + hashes = [ + "h1:4ht2uLwmZhnSgxD0eFaPdacz11yUEZ0N3vUfP1mnirE=", + "h1:B/T/w2S/i2Ififrco0tlHU3IvclmbRb/J0sTc6hqG88=", + "h1:FajCus3E8nqYFGk7VC1l0vRQs4VoL9H5E6x1d+CpGaY=", + "h1:IWydTb5OzjXV1KqKTkc+7fNG3chGHhpW6/N26l8KmTs=", + "h1:K9fFFOcQTYS4GaBKShNN7vKZTysfOLB/qrEyF9ngPLw=", + "h1:QKxfDx4AoGdFv59xU7ZFM9E8c7V4LnfbPhjKF3+WpVc=", + "h1:UUXVJJ8j4WnO2IJO24jeLkGjiEAO/ILCCBqOPHOguNM=", + "h1:Us9liS1XzeTcb9+MHOem8IfsirTam38jpx1xhqRp3nM=", + "h1:kc+lpSApn2IBG1POfZWLhUPmH8Ra728Q6uv+Zdj31cc=", + "h1:nvbEfb1kzj5Zovvk9EdfE2Nc7oOUtLLSpDMMXntbZqQ=", + "h1:ra4YUZTi7TxKZ3hlo6IzWbReEspj2mwpqgvv9Eo9vTk=", + "h1:uhnwCNFGvURyqB3XSj5ISd9sXF195pk52u11KrzOH9A=", + "h1:vkNm99bZXAeVba6ttcnYkkr5u1sEJtM/NeazdGNYsR8=", + "zh:19f7fdc47921acfdf629abcaba66a1857bd8f97173124b5610002dcf1437fe3f", + "zh:26638a04481a13030ad03f35d0e583d915bbca3c1ea8c00254292d3fddec4745", + "zh:2967ffecbdc31e74d34584cb62b3641cf696bfa4a61e5caeeda34d0beb6a1321", + "zh:4e3f82026beb5ab78b4a17651dcf89ef4af618ec04de613149d8e5ff3dabd965", + "zh:531a44d9e278a7afb179a714a45e789efbd86cb136e6e97d94cb72fb81df1a5b", + "zh:649596a782c9b4d9452e278c016c5565d853561a15bca1c9d2f74a651db212ad", + "zh:69521c7a263ddb6779875ea065a0cbdd52f89832137abf020c6949d07a25e7b0", + "zh:7f26ad680c16e6119b5b5ad54d3720b883ef748a23fc295848f16407b08574c3", + "zh:8bab462c26fcfb2907b1c06dc28839c90cb90be92b5e3eb4e34a127562f1e840", + "zh:98e19b8293c4ef1b797ca688b8602a97eda4b2cd65a0f6bfc4a56ac9e126d3d7", + "zh:9ba7771a967f517af8b7ea61255a40b3f3e30e08fdc21ff09028b93daa765799", + "zh:9f1db2d89bf586d387e0d70ea71cbfea6a6da6d0715ea8de501b4830bb5df114", + "zh:9f37804849b3a5996399f0879024a24c4c6dbd2d08ee021ed9b14dffa5a409a2", + "zh:a45ff9ef05020fd12f2e78b44d5a366cd20cca7dffe1b6e777c9e9760966ed57", + "zh:cf56438233a38f2cfcb5d367ddb08c54c24ad046a3ff9016b65538a299b9777c", + "zh:d19cd53020dbce893283b070ba355ee29c089496b98341dbf72ebecf5595d499", + "zh:f3cacc1f46e79730cbe1ee26935053e889cb37ec4721899501478a3548d52aea", + "zh:f98e7cbf283c02073155136a073298e6ea4a10924cb185fbaf7e601a8fd6bd57", + "zh:fe1f9ddfc2fc956d41492ec99d5165304ecd25c83bbfbec11e804269364e298c", + ] +} diff --git a/deployment/modules/grafana/cluster/config.tf b/deployment/modules/grafana/cluster/config.tf new file mode 100644 index 0000000..3e00c7c --- /dev/null +++ b/deployment/modules/grafana/cluster/config.tf @@ -0,0 +1,14 @@ +terraform { + required_version = "~> 1.10" + + required_providers { + grafana = { + source = "grafana/grafana" + version = "~> 4.46" + } + onepassword = { + source = "1Password/onepassword" + version = "~> 3.3" + } + } +} diff --git a/deployment/modules/grafana/cluster/mise.toml b/deployment/modules/grafana/cluster/mise.toml new file mode 100644 index 0000000..5c5c5b3 --- /dev/null +++ b/deployment/modules/grafana/cluster/mise.toml @@ -0,0 +1,15 @@ +[tasks.init] +extends = "tg:init" + +[tasks.plan] +extends = "tg:plan" + +[tasks.apply] +extends = "tg:apply" + +[tasks.destroy] +extends = "tg:destroy" +wait_for = ["//deployment/modules/ovh/account:destroy"] + +[tasks.output] +extends = "tg:output" diff --git a/deployment/modules/grafana/cluster/onepassword.tf b/deployment/modules/grafana/cluster/onepassword.tf new file mode 100644 index 0000000..e9de941 --- /dev/null +++ b/deployment/modules/grafana/cluster/onepassword.tf @@ -0,0 +1,23 @@ +# Reads the Grafana admin password the cluster's ExternalSecret also consumes, +# and publishes the Rootly service account token next to it so the Rootly +# integration can be configured from the vault rather than from kubectl. +locals { + # Mirrors the ENVIRONMENT_SHORT mapping in .mise/config.toml. + env_short = var.env == "production" ? "prod" : var.env == "development" ? "dev" : var.env +} + +data "onepassword_vault" "env" { + name = "o11y_tf_${local.env_short}" +} + +data "onepassword_item" "grafana_admin_password" { + vault = data.onepassword_vault.env.uuid + title = "GRAFANA_ADMIN_PASSWORD" +} + +resource "onepassword_item" "rootly_service_account_token" { + vault = data.onepassword_vault.env.uuid + title = "ROOTLY_GRAFANA_SERVICE_ACCOUNT_TOKEN" + category = "password" + password = grafana_service_account_token.rootly.key +} diff --git a/deployment/modules/grafana/cluster/outputs.tf b/deployment/modules/grafana/cluster/outputs.tf new file mode 100644 index 0000000..65c6091 --- /dev/null +++ b/deployment/modules/grafana/cluster/outputs.tf @@ -0,0 +1,12 @@ +output "grafana_url" { + value = local.grafana_url +} + +output "rootly_service_account_id" { + value = grafana_service_account.rootly.id +} + +output "rootly_service_account_token" { + value = grafana_service_account_token.rootly.key + sensitive = true +} diff --git a/deployment/modules/grafana/cluster/providers.tf b/deployment/modules/grafana/cluster/providers.tf new file mode 100644 index 0000000..ebcf3ba --- /dev/null +++ b/deployment/modules/grafana/cluster/providers.tf @@ -0,0 +1,9 @@ +provider "grafana" { + url = local.grafana_url + auth = "${var.grafana_admin_user}:${data.onepassword_item.grafana_admin_password.password}" +} + +provider "onepassword" { + connect_url = var.op_connect_host + connect_token = var.op_connect_token_write +} diff --git a/deployment/modules/grafana/cluster/rootly.tf b/deployment/modules/grafana/cluster/rootly.tf new file mode 100644 index 0000000..0df996f --- /dev/null +++ b/deployment/modules/grafana/cluster/rootly.tf @@ -0,0 +1,22 @@ +# Service account for Rootly's Grafana integration (Integrations > Grafana in +# Rootly), which has no API or Terraform surface: it is installed by hand with +# this cluster's Grafana URL and the token below, read from the env vault. +# Rootly requires Admin to take dashboard snapshots during incidents. +# +# This lives apart from the rootly/cluster module on purpose: that module must +# stay plannable while Grafana is down, since Grafana being down is what its +# heartbeat reports. +locals { + grafana_url = var.env == "production" ? "https://grafana.futostatus.com" : "https://grafana.${var.env}.futostatus.com" +} + +resource "grafana_service_account" "rootly" { + name = "rootly" + role = "Admin" + is_disabled = false +} + +resource "grafana_service_account_token" "rootly" { + name = "rootly-integration" + service_account_id = grafana_service_account.rootly.id +} diff --git a/deployment/modules/grafana/cluster/terragrunt.hcl b/deployment/modules/grafana/cluster/terragrunt.hcl new file mode 100644 index 0000000..6f1f27c --- /dev/null +++ b/deployment/modules/grafana/cluster/terragrunt.hcl @@ -0,0 +1,43 @@ +terraform { + source = "." + + extra_arguments custom_vars { + commands = get_terraform_commands_that_need_vars() + + # The onepassword provider treats OP_SERVICE_ACCOUNT_TOKEN as a configured + # credential alongside the explicit Connect creds and errors on the conflict. + # CI sets it for `op run` itself; blank it for the tofu child process only. + env_vars = { + OP_SERVICE_ACCOUNT_TOKEN = "" + } + } +} + +locals { + env = get_env("TF_VAR_env") + stage = get_env("TF_VAR_stage") +} + +generate "backend" { + path = "backend.tf" + if_exists = "overwrite_terragrunt" + contents = < +# endpoint with the secret as a bearer token; the notification title becomes +# the alert summary and commonLabels become alert labels. +locals { + projects = toset(["o11y", "yucca", "fmeet", "harbor", "fip"]) + + project_service_ids = merge( + { o11y = rootly_service.o11y.id }, + { for project, service in rootly_service.project : project => service.id }, + ) + + grafana_webhooks_base = "https://webhooks.rootly.com/webhooks/incoming/grafana_webhooks" +} + +data "rootly_alert_urgency" "medium" { + name = "Medium" +} + +data "rootly_alert_urgency" "low" { + name = "Low" +} + +resource "rootly_service" "project" { + for_each = setsubtract(local.projects, ["o11y"]) + name = "${each.key}-${var.env}" + description = "${each.key} (${var.env}): alerts raised by the o11y Grafana from the ${each.key} folder." + environment_ids = [rootly_environment.env.id] +} + +# Urgency follows the rule's severity label (critical -> High, warning -> +# Medium); anything unlabelled lands on Low. +resource "rootly_alerts_source" "grafana" { + for_each = local.projects + name = "${each.key}-${var.env}-grafana" + source_type = "grafana" + alert_urgency_id = data.rootly_alert_urgency.low.id + + alert_source_urgency_rules_attributes { + kind = "payload" + json_path = "$.commonLabels.severity" + operator = "is" + value = "critical" + alert_urgency_id = data.rootly_alert_urgency.high.id + } + + alert_source_urgency_rules_attributes { + kind = "payload" + json_path = "$.commonLabels.severity" + operator = "is" + value = "warning" + alert_urgency_id = data.rootly_alert_urgency.medium.id + } +} diff --git a/deployment/modules/rootly/cluster/onepassword.tf b/deployment/modules/rootly/cluster/onepassword.tf index 9a9c6aa..f17ddfc 100644 --- a/deployment/modules/rootly/cluster/onepassword.tf +++ b/deployment/modules/rootly/cluster/onepassword.tf @@ -27,3 +27,21 @@ resource "onepassword_item" "heartbeat_ping_secret" { category = "password" password = rootly_heartbeat.grafana_alerting.secret } + +# Per-project Grafana alert source credentials, consumed by the rootly-alerts-* +# ExternalSecrets that feed the rootly- contact points. +resource "onepassword_item" "grafana_alerts_url" { + for_each = local.projects + vault = data.onepassword_vault.env.uuid + title = "ROOTLY_ALERTS_${upper(each.key)}_URL" + category = "password" + password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}" +} + +resource "onepassword_item" "grafana_alerts_secret" { + for_each = local.projects + vault = data.onepassword_vault.env.uuid + title = "ROOTLY_ALERTS_${upper(each.key)}_SECRET" + category = "password" + password = rootly_alerts_source.grafana[each.key].secret +} diff --git a/deployment/modules/rootly/cluster/outputs.tf b/deployment/modules/rootly/cluster/outputs.tf index 4de2b44..e3a124d 100644 --- a/deployment/modules/rootly/cluster/outputs.tf +++ b/deployment/modules/rootly/cluster/outputs.tf @@ -10,3 +10,11 @@ output "heartbeat_secret" { value = rootly_heartbeat.grafana_alerting.secret sensitive = true } + +output "grafana_alert_sources" { + value = { for project, source in rootly_alerts_source.grafana : project => source.id } +} + +output "project_service_ids" { + value = local.project_service_ids +} diff --git a/deployment/modules/rootly/cluster/rootly.tf b/deployment/modules/rootly/cluster/rootly.tf index f9fb11a..351b59f 100644 --- a/deployment/modules/rootly/cluster/rootly.tf +++ b/deployment/modules/rootly/cluster/rootly.tf @@ -28,14 +28,15 @@ resource "rootly_heartbeat" "grafana_alerting" { enabled = true } -# Deliver Rootly alerts for this service to the env's Discord channel. Interim -# receiver until escalation policies exist; Rootly cloud -> Discord, independent -# of the cluster whose death the heartbeat reports. +# Deliver Rootly alerts on every project service (heartbeat and Grafana alert +# sources alike) to the env's Discord channel. Interim receiver until +# escalation policies exist; Rootly cloud -> Discord, independent of the +# cluster whose death the heartbeat reports. resource "rootly_workflow_alert" "discord_fired" { name = "o11y-${var.env}-alert-fired-to-discord" - description = "Posts new alerts on the o11y-${var.env} service to the ${var.env} Discord channel." + description = "Posts new alerts on the ${var.env} project services to the ${var.env} Discord channel." enabled = true - service_ids = [rootly_service.o11y.id] + service_ids = values(local.project_service_ids) trigger_params { triggers = ["alert_created"] } @@ -51,7 +52,7 @@ resource "rootly_workflow_task_http_client" "discord_fired" { body = jsonencode({ username = "Rootly" avatar_url = "https://avatars.githubusercontent.com/u/78240982" - content = "🔴 **Rootly** (o11y-${var.env}): {{ alert.summary }}" + content = "🔴 **Rootly** (${var.env}): {{ alert.summary }}" }) succeed_on_status = "200|204" retry_count = "4" @@ -61,9 +62,9 @@ resource "rootly_workflow_task_http_client" "discord_fired" { resource "rootly_workflow_alert" "discord_resolved" { name = "o11y-${var.env}-alert-resolved-to-discord" - description = "Posts alert resolutions on the o11y-${var.env} service to the ${var.env} Discord channel." + description = "Posts alert resolutions on the ${var.env} project services to the ${var.env} Discord channel." enabled = true - service_ids = [rootly_service.o11y.id] + service_ids = values(local.project_service_ids) trigger_params { triggers = ["alert_status_updated"] alert_condition_status = "IS" @@ -81,7 +82,7 @@ resource "rootly_workflow_task_http_client" "discord_resolved" { body = jsonencode({ username = "Rootly" avatar_url = "https://avatars.githubusercontent.com/u/78240982" - content = "🟢 **Rootly** (o11y-${var.env}): resolved — {{ alert.summary }}" + content = "🟢 **Rootly** (${var.env}): resolved — {{ alert.summary }}" }) succeed_on_status = "200|204" retry_count = "4" diff --git a/docs/06-dashboards-and-alerts-guide.md b/docs/06-dashboards-and-alerts-guide.md index 3094e38..0d85f43 100644 --- a/docs/06-dashboards-and-alerts-guide.md +++ b/docs/06-dashboards-and-alerts-guide.md @@ -126,18 +126,20 @@ Cluster-generic boards (Kubernetes views and system, node exporter, vmagent, Cil ## Alerting -**Contact points.** A `GrafanaContactPoint` per destination. Secrets (like a Discord webhook) come from a Secret via `receivers[].valuesFrom`, populated by an ExternalSecret from 1Password - never in git. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica. +**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own bearer secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's URL and secret into the env 1Password vault (`ROOTLY_ALERTS__URL` / `_SECRET`); an ExternalSecret materializes them into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica. + +**Rootly's Grafana integration.** Besides the alert sources, Rootly has an account-level Grafana integration (Integrations > Grafana) that takes this cluster's Grafana URL and an Admin service account token; it is what lets Rootly deep-link rules and snapshot dashboards into incidents. Rootly exposes no API or Terraform surface for it, so it is installed by hand once per env. The `deployment/modules/grafana/cluster` module owns the `rootly` service account and its token and writes the token to the env vault as `ROOTLY_GRAFANA_SERVICE_ACCOUNT_TOKEN`; paste that and `https://grafana.` into Rootly. It is a separate module from `rootly/cluster` so the latter stays plannable while Grafana is down. **Routing.** One `GrafanaNotificationPolicy` routes by the **`grafana_folder`** label - which Grafana adds automatically from the folder each rule files into, so routing follows the folder (the project boundary) with no label to maintain: ```yaml route: - receiver: discord # default / catch-all + receiver: rootly-o11y # default / catch-all for folders without a source of their own routes: - object_matchers: [["grafana_folder", "=", "yucca"]] - receiver: discord + receiver: rootly-yucca - object_matchers: [["grafana_folder", "=", "o11y"]] - receiver: discord # point at an o11y-specific contact point when one exists + receiver: rootly-o11y ``` So **routing follows the folder automatically** - no per-rule label to set or keep in sync. (Existing rules still carry a `project` *rule* label; it is legacy and unused for routing — distinct from the `project` *series* label every shipper stamps, see the [shipping guide](05-shipping-metrics-guide.md#labels).) Notifications additionally group by `cluster` (alongside `grafana_folder` and `alertname`), so the same rule firing in two clusters arrives as two grouped notifications rather than one blended message. @@ -150,7 +152,7 @@ So **routing follows the folder automatically** - no per-rule label to set or ke 1. Pick a delivery model: **Model A** (recommended for a separate repo/cluster - you own a signed bundle, o11y adds one OCIRepository) or **Model B** (PR the CRs into `base/grafana`). 2. Everything you ship files under **your project's folder**; ask for one if it does not exist. -3. Routing follows your folder automatically; add a route matching your `grafana_folder` (and, if you want your own channel, a contact point). +3. Routing follows your folder automatically; ask for your project to be added to the Rootly module's `projects` list, which gives you a Rootly alert source, a service, and the `rootly-` contact point and route in `base/grafana/app`. 4. Dashboards use a `$datasource` variable and map `DS_PROMETHEUS` to `VictoriaMetrics`; alerts query the `VictoriaMetrics` datasource. Anything that must see other clusters' series uses `VictoriaMetrics Fleet` instead (see Datasources and tenants). 5. Tag dashboards by signal/layer in the JSON (`metrics`, `logs`, `infra`, ...) so they stay filterable across folders (see Tags). Alerts that compare across clusters aggregate `by (cluster)`; stamp the five identity labels on your series (see the [shipping guide](05-shipping-metrics-guide.md#labels)) so per-cluster alerting works. diff --git a/kubernetes/apps/base/grafana/app/contactpoint-discord.yaml b/kubernetes/apps/base/grafana/app/contactpoint-discord.yaml deleted file mode 100644 index 32be671..0000000 --- a/kubernetes/apps/base/grafana/app/contactpoint-discord.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaContactPoint -metadata: - name: discord -spec: - resyncPeriod: 1h - instanceSelector: - matchLabels: - dashboards: grafana - name: discord - receivers: - - type: discord - settings: - use_embed_description: true - title: |- - {{ if .Alerts.Firing }}🔴 {{ .Alerts.Firing | len }} firing{{ else }}✅ resolved{{ end }} · {{ .CommonLabels.grafana_folder }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} - message: |- - {{ range .Alerts -}} - **{{ .Labels.alertname }}**{{ if and .Labels.severity (ne .Labels.severity "none") }} `{{ .Labels.severity }}`{{ end }} - {{ if .Annotations.summary }}> {{ .Annotations.summary }}{{ if match "67" .Annotations.summary }} {{ end }} - {{ end }}[View rule]({{ .GeneratorURL }}){{ if .SilenceURL }} · [Silence]({{ .SilenceURL }}){{ end }} - - {{ end -}} - valuesFrom: - - targetPath: url - valueFrom: - secretKeyRef: - name: grafana-discord-webhook - key: url diff --git a/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml b/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml new file mode 100644 index 0000000..77a2286 --- /dev/null +++ b/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml @@ -0,0 +1,154 @@ +--- +# One webhook contact point per Grafana folder, each posting to that project's +# Rootly alert source (its own secret) on the project's service. The +# notification title becomes the Rootly alert summary, so it stays stable +# between the firing and resolved notifications of the same group. +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaContactPoint +metadata: + name: rootly-o11y +spec: + resyncPeriod: 1h + instanceSelector: + matchLabels: + dashboards: grafana + name: rootly-o11y + receivers: + - type: webhook + settings: + httpMethod: POST + authorization_scheme: Bearer + title: |- + {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} + valuesFrom: + - targetPath: url + valueFrom: + secretKeyRef: + name: rootly-alerts-o11y + key: url + - targetPath: authorization_credentials + valueFrom: + secretKeyRef: + name: rootly-alerts-o11y + key: token +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaContactPoint +metadata: + name: rootly-yucca +spec: + resyncPeriod: 1h + instanceSelector: + matchLabels: + dashboards: grafana + name: rootly-yucca + receivers: + - type: webhook + settings: + httpMethod: POST + authorization_scheme: Bearer + title: |- + {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} + valuesFrom: + - targetPath: url + valueFrom: + secretKeyRef: + name: rootly-alerts-yucca + key: url + - targetPath: authorization_credentials + valueFrom: + secretKeyRef: + name: rootly-alerts-yucca + key: token +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaContactPoint +metadata: + name: rootly-fmeet +spec: + resyncPeriod: 1h + instanceSelector: + matchLabels: + dashboards: grafana + name: rootly-fmeet + receivers: + - type: webhook + settings: + httpMethod: POST + authorization_scheme: Bearer + title: |- + {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} + valuesFrom: + - targetPath: url + valueFrom: + secretKeyRef: + name: rootly-alerts-fmeet + key: url + - targetPath: authorization_credentials + valueFrom: + secretKeyRef: + name: rootly-alerts-fmeet + key: token +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaContactPoint +metadata: + name: rootly-harbor +spec: + resyncPeriod: 1h + instanceSelector: + matchLabels: + dashboards: grafana + name: rootly-harbor + receivers: + - type: webhook + settings: + httpMethod: POST + authorization_scheme: Bearer + title: |- + {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} + valuesFrom: + - targetPath: url + valueFrom: + secretKeyRef: + name: rootly-alerts-harbor + key: url + - targetPath: authorization_credentials + valueFrom: + secretKeyRef: + name: rootly-alerts-harbor + key: token +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaContactPoint +metadata: + name: rootly-fip +spec: + resyncPeriod: 1h + instanceSelector: + matchLabels: + dashboards: grafana + name: rootly-fip + receivers: + - type: webhook + settings: + httpMethod: POST + authorization_scheme: Bearer + title: |- + {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} + valuesFrom: + - targetPath: url + valueFrom: + secretKeyRef: + name: rootly-alerts-fip + key: url + - targetPath: authorization_credentials + valueFrom: + secretKeyRef: + name: rootly-alerts-fip + key: token diff --git a/kubernetes/apps/base/grafana/app/externalsecret.yaml b/kubernetes/apps/base/grafana/app/externalsecret.yaml index 41c0672..01f09c3 100644 --- a/kubernetes/apps/base/grafana/app/externalsecret.yaml +++ b/kubernetes/apps/base/grafana/app/externalsecret.yaml @@ -74,16 +74,93 @@ spec: apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: grafana-discord-webhook + name: rootly-alerts-o11y spec: secretStoreRef: kind: ClusterSecretStore name: onepassword-environment target: - name: grafana-discord-webhook - template: - data: - url: "{{ .password }}" - dataFrom: - - extract: - key: GRAFANA_DISCORD_WEBHOOK + name: rootly-alerts-o11y + data: + - secretKey: url + remoteRef: + key: ROOTLY_ALERTS_O11Y_URL + - secretKey: token + remoteRef: + key: ROOTLY_ALERTS_O11Y_SECRET +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: rootly-alerts-yucca +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-environment + target: + name: rootly-alerts-yucca + data: + - secretKey: url + remoteRef: + key: ROOTLY_ALERTS_YUCCA_URL + - secretKey: token + remoteRef: + key: ROOTLY_ALERTS_YUCCA_SECRET +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: rootly-alerts-fmeet +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-environment + target: + name: rootly-alerts-fmeet + data: + - secretKey: url + remoteRef: + key: ROOTLY_ALERTS_FMEET_URL + - secretKey: token + remoteRef: + key: ROOTLY_ALERTS_FMEET_SECRET +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: rootly-alerts-harbor +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-environment + target: + name: rootly-alerts-harbor + data: + - secretKey: url + remoteRef: + key: ROOTLY_ALERTS_HARBOR_URL + - secretKey: token + remoteRef: + key: ROOTLY_ALERTS_HARBOR_SECRET +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: rootly-alerts-fip +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-environment + target: + name: rootly-alerts-fip + data: + - secretKey: url + remoteRef: + key: ROOTLY_ALERTS_FIP_URL + - secretKey: token + remoteRef: + key: ROOTLY_ALERTS_FIP_SECRET diff --git a/kubernetes/apps/base/grafana/app/kustomization.yaml b/kubernetes/apps/base/grafana/app/kustomization.yaml index 47200c3..242ed5c 100644 --- a/kubernetes/apps/base/grafana/app/kustomization.yaml +++ b/kubernetes/apps/base/grafana/app/kustomization.yaml @@ -8,8 +8,8 @@ resources: - ./datasource-victoria-logs-legacy.yaml - ./datasource-fleet.yaml - ./datasource-victoriametrics.yaml - - ./contactpoint-discord.yaml - ./contactpoint-rootly.yaml + - ./contactpoint-rootly-alerts.yaml - ./notificationpolicy.yaml - ./servicemonitor.yaml - ./alerts-o11y.yaml diff --git a/kubernetes/apps/base/grafana/app/notificationpolicy.yaml b/kubernetes/apps/base/grafana/app/notificationpolicy.yaml index 9182399..9179b15 100644 --- a/kubernetes/apps/base/grafana/app/notificationpolicy.yaml +++ b/kubernetes/apps/base/grafana/app/notificationpolicy.yaml @@ -10,7 +10,8 @@ spec: matchLabels: dashboards: grafana route: - receiver: discord + # Folders without a Rootly alert source of their own land on the o11y one. + receiver: rootly-o11y group_by: - grafana_folder - alertname @@ -25,18 +26,18 @@ spec: group_wait: 15s group_interval: 2m repeat_interval: 2m - - receiver: discord - object_matchers: - - ["grafana_folder", "=", "yucca"] - - receiver: discord + - receiver: rootly-o11y object_matchers: - ["grafana_folder", "=", "o11y"] - - receiver: discord + - receiver: rootly-yucca + object_matchers: + - ["grafana_folder", "=", "yucca"] + - receiver: rootly-fmeet object_matchers: - ["grafana_folder", "=", "fmeet"] - - receiver: discord + - receiver: rootly-harbor object_matchers: - ["grafana_folder", "=", "harbor"] - - receiver: discord + - receiver: rootly-fip object_matchers: - ["grafana_folder", "=", "fip"]