diff --git a/deployment/modules/rootly/cluster/alerts.tf b/deployment/modules/rootly/cluster/alerts.tf index a016955..b337624 100644 --- a/deployment/modules/rootly/cluster/alerts.tf +++ b/deployment/modules/rootly/cluster/alerts.tf @@ -4,8 +4,9 @@ # status-page items. The o11y folder reuses the service the heartbeat targets. # # Grafana posts each grouped notification to the source's /notify/Service/ -# endpoint with the secret as a bearer token; the notification title becomes -# the alert summary and commonLabels become alert labels. +# endpoint with the source secret as a query parameter; the notification title +# becomes the alert summary, commonLabels become alert labels, and a resolved +# notification resolves the alert. locals { projects = toset(["o11y", "yucca", "fmeet", "harbor", "fip"]) diff --git a/deployment/modules/rootly/cluster/onepassword.tf b/deployment/modules/rootly/cluster/onepassword.tf index f17ddfc..05770df 100644 --- a/deployment/modules/rootly/cluster/onepassword.tf +++ b/deployment/modules/rootly/cluster/onepassword.tf @@ -29,19 +29,14 @@ resource "onepassword_item" "heartbeat_ping_secret" { } # Per-project Grafana alert source credentials, consumed by the rootly-alerts-* -# ExternalSecrets that feed the rootly- contact points. +# ExternalSecrets that feed the rootly- contact points. The secret +# rides in the URL: Rootly's Grafana endpoint only reads it from the `secret` +# query parameter and answers 404 "integration cannot be found" to a bearer +# header, so the URL item is the whole credential. resource "onepassword_item" "grafana_alerts_url" { for_each = local.projects vault = data.onepassword_vault.env.uuid title = "ROOTLY_ALERTS_${upper(each.key)}_URL" category = "password" - password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}" -} - -resource "onepassword_item" "grafana_alerts_secret" { - for_each = local.projects - vault = data.onepassword_vault.env.uuid - title = "ROOTLY_ALERTS_${upper(each.key)}_SECRET" - category = "password" - password = rootly_alerts_source.grafana[each.key].secret + password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}?secret=${rootly_alerts_source.grafana[each.key].secret}" } diff --git a/docs/06-dashboards-and-alerts-guide.md b/docs/06-dashboards-and-alerts-guide.md index 0d85f43..178220d 100644 --- a/docs/06-dashboards-and-alerts-guide.md +++ b/docs/06-dashboards-and-alerts-guide.md @@ -126,7 +126,7 @@ Cluster-generic boards (Kubernetes views and system, node exporter, vmagent, Cil ## Alerting -**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own bearer secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's URL and secret into the env 1Password vault (`ROOTLY_ALERTS__URL` / `_SECRET`); an ExternalSecret materializes them into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica. +**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's webhook URL into the env 1Password vault as `ROOTLY_ALERTS__URL`; the secret rides in that URL's `secret` query parameter because Rootly's Grafana endpoint reads it nowhere else. An ExternalSecret materializes it into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. A resolved Grafana notification resolves the Rootly alert. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica. **Rootly's Grafana integration.** Besides the alert sources, Rootly has an account-level Grafana integration (Integrations > Grafana) that takes this cluster's Grafana URL and an Admin service account token; it is what lets Rootly deep-link rules and snapshot dashboards into incidents. Rootly exposes no API or Terraform surface for it, so it is installed by hand once per env. The `deployment/modules/grafana/cluster` module owns the `rootly` service account and its token and writes the token to the env vault as `ROOTLY_GRAFANA_SERVICE_ACCOUNT_TOKEN`; paste that and `https://grafana.` into Rootly. It is a separate module from `rootly/cluster` so the latter stays plannable while Grafana is down. diff --git a/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml b/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml index 77a2286..a922d25 100644 --- a/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml +++ b/kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml @@ -1,8 +1,9 @@ --- # One webhook contact point per Grafana folder, each posting to that project's -# Rootly alert source (its own secret) on the project's service. The -# notification title becomes the Rootly alert summary, so it stays stable -# between the firing and resolved notifications of the same group. +# Rootly alert source on the project's service. The URL carries the source +# secret as a query parameter (Rootly reads it nowhere else). The notification +# title becomes the Rootly alert summary, so it stays stable between the +# firing and resolved notifications of the same group. # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaContactPoint @@ -18,7 +19,6 @@ spec: - type: webhook settings: httpMethod: POST - authorization_scheme: Bearer title: |- {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} valuesFrom: @@ -27,11 +27,6 @@ spec: secretKeyRef: name: rootly-alerts-o11y key: url - - targetPath: authorization_credentials - valueFrom: - secretKeyRef: - name: rootly-alerts-o11y - key: token --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json apiVersion: grafana.integreatly.org/v1beta1 @@ -48,7 +43,6 @@ spec: - type: webhook settings: httpMethod: POST - authorization_scheme: Bearer title: |- {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} valuesFrom: @@ -57,11 +51,6 @@ spec: secretKeyRef: name: rootly-alerts-yucca key: url - - targetPath: authorization_credentials - valueFrom: - secretKeyRef: - name: rootly-alerts-yucca - key: token --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json apiVersion: grafana.integreatly.org/v1beta1 @@ -78,7 +67,6 @@ spec: - type: webhook settings: httpMethod: POST - authorization_scheme: Bearer title: |- {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} valuesFrom: @@ -87,11 +75,6 @@ spec: secretKeyRef: name: rootly-alerts-fmeet key: url - - targetPath: authorization_credentials - valueFrom: - secretKeyRef: - name: rootly-alerts-fmeet - key: token --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json apiVersion: grafana.integreatly.org/v1beta1 @@ -108,7 +91,6 @@ spec: - type: webhook settings: httpMethod: POST - authorization_scheme: Bearer title: |- {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} valuesFrom: @@ -117,11 +99,6 @@ spec: secretKeyRef: name: rootly-alerts-harbor key: url - - targetPath: authorization_credentials - valueFrom: - secretKeyRef: - name: rootly-alerts-harbor - key: token --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json apiVersion: grafana.integreatly.org/v1beta1 @@ -138,7 +115,6 @@ spec: - type: webhook settings: httpMethod: POST - authorization_scheme: Bearer title: |- {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }} valuesFrom: @@ -147,8 +123,3 @@ spec: secretKeyRef: name: rootly-alerts-fip key: url - - targetPath: authorization_credentials - valueFrom: - secretKeyRef: - name: rootly-alerts-fip - key: token diff --git a/kubernetes/apps/base/grafana/app/externalsecret.yaml b/kubernetes/apps/base/grafana/app/externalsecret.yaml index 01f09c3..a05dcb6 100644 --- a/kubernetes/apps/base/grafana/app/externalsecret.yaml +++ b/kubernetes/apps/base/grafana/app/externalsecret.yaml @@ -85,9 +85,6 @@ spec: - secretKey: url remoteRef: key: ROOTLY_ALERTS_O11Y_URL - - secretKey: token - remoteRef: - key: ROOTLY_ALERTS_O11Y_SECRET --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json apiVersion: external-secrets.io/v1 @@ -104,9 +101,6 @@ spec: - secretKey: url remoteRef: key: ROOTLY_ALERTS_YUCCA_URL - - secretKey: token - remoteRef: - key: ROOTLY_ALERTS_YUCCA_SECRET --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json apiVersion: external-secrets.io/v1 @@ -123,9 +117,6 @@ spec: - secretKey: url remoteRef: key: ROOTLY_ALERTS_FMEET_URL - - secretKey: token - remoteRef: - key: ROOTLY_ALERTS_FMEET_SECRET --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json apiVersion: external-secrets.io/v1 @@ -142,9 +133,6 @@ spec: - secretKey: url remoteRef: key: ROOTLY_ALERTS_HARBOR_URL - - secretKey: token - remoteRef: - key: ROOTLY_ALERTS_HARBOR_SECRET --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json apiVersion: external-secrets.io/v1 @@ -161,6 +149,3 @@ spec: - secretKey: url remoteRef: key: ROOTLY_ALERTS_FIP_URL - - secretKey: token - remoteRef: - key: ROOTLY_ALERTS_FIP_SECRET diff --git a/kubernetes/apps/base/grafana/app/notificationpolicy.yaml b/kubernetes/apps/base/grafana/app/notificationpolicy.yaml index 9179b15..e0406ed 100644 --- a/kubernetes/apps/base/grafana/app/notificationpolicy.yaml +++ b/kubernetes/apps/base/grafana/app/notificationpolicy.yaml @@ -35,9 +35,11 @@ spec: - receiver: rootly-fmeet object_matchers: - ["grafana_folder", "=", "fmeet"] + # The harbor bundle titles its folder "Harbor"; grafana_folder carries the + # title, so match it case-insensitively rather than by the CR name. - receiver: rootly-harbor object_matchers: - - ["grafana_folder", "=", "harbor"] + - ["grafana_folder", "=~", "(?i)^harbor$"] - receiver: rootly-fip object_matchers: - ["grafana_folder", "=", "fip"]