fix(rootly): pass the Grafana source secret as a query parameter (#343)

deployment/modules/rootly/cluster/alerts.tf

@@ -4,8 +4,9 @@
 # status-page items. The o11y folder reuses the service the heartbeat targets.
 #
 # Grafana posts each grouped notification to the source's /notify/Service/<id>
-# endpoint with the secret as a bearer token; the notification title becomes
+# endpoint with the source secret as a query parameter; the notification title
-# the alert summary and commonLabels become alert labels.
+# becomes the alert summary, commonLabels become alert labels, and a resolved
+# notification resolves the alert.
 locals {
   projects = toset(["o11y", "yucca", "fmeet", "harbor", "fip"])
 

deployment/modules/rootly/cluster/onepassword.tf

@@ -29,19 +29,14 @@ resource "onepassword_item" "heartbeat_ping_secret" {
 }
 
 # Per-project Grafana alert source credentials, consumed by the rootly-alerts-*
-# ExternalSecrets that feed the rootly-<project> contact points.
+# ExternalSecrets that feed the rootly-<project> contact points. The secret
+# rides in the URL: Rootly's Grafana endpoint only reads it from the `secret`
+# query parameter and answers 404 "integration cannot be found" to a bearer
+# header, so the URL item is the whole credential.
 resource "onepassword_item" "grafana_alerts_url" {
   for_each = local.projects
   vault    = data.onepassword_vault.env.uuid
   title    = "ROOTLY_ALERTS_${upper(each.key)}_URL"
   category = "password"
-  password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}"
+  password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}?secret=${rootly_alerts_source.grafana[each.key].secret}"
-}
-
-resource "onepassword_item" "grafana_alerts_secret" {
-  for_each = local.projects
-  vault    = data.onepassword_vault.env.uuid
-  title    = "ROOTLY_ALERTS_${upper(each.key)}_SECRET"
-  category = "password"
-  password = rootly_alerts_source.grafana[each.key].secret
 }

docs/06-dashboards-and-alerts-guide.md

@@ -126,7 +126,7 @@ Cluster-generic boards (Kubernetes views and system, node exporter, vmagent, Cil
 
 ## Alerting
 
-**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-<project>` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own bearer secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's URL and secret into the env 1Password vault (`ROOTLY_ALERTS_<PROJECT>_URL` / `_SECRET`); an ExternalSecret materializes them into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica.
+**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-<project>` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's webhook URL into the env 1Password vault as `ROOTLY_ALERTS_<PROJECT>_URL`; the secret rides in that URL's `secret` query parameter because Rootly's Grafana endpoint reads it nowhere else. An ExternalSecret materializes it into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. A resolved Grafana notification resolves the Rootly alert. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica.
 
 **Rootly's Grafana integration.** Besides the alert sources, Rootly has an account-level Grafana integration (Integrations > Grafana) that takes this cluster's Grafana URL and an Admin service account token; it is what lets Rootly deep-link rules and snapshot dashboards into incidents. Rootly exposes no API or Terraform surface for it, so it is installed by hand once per env. The `deployment/modules/grafana/cluster` module owns the `rootly` service account and its token and writes the token to the env vault as `ROOTLY_GRAFANA_SERVICE_ACCOUNT_TOKEN`; paste that and `https://grafana.<env domain>` into Rootly. It is a separate module from `rootly/cluster` so the latter stays plannable while Grafana is down.
 

kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml

@@ -1,8 +1,9 @@
 ---
 # One webhook contact point per Grafana folder, each posting to that project's
-# Rootly alert source (its own secret) on the project's service. The
+# Rootly alert source on the project's service. The URL carries the source
-# notification title becomes the Rootly alert summary, so it stays stable
+# secret as a query parameter (Rootly reads it nowhere else). The notification
-# between the firing and resolved notifications of the same group.
+# title becomes the Rootly alert summary, so it stays stable between the
+# firing and resolved notifications of the same group.
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
 apiVersion: grafana.integreatly.org/v1beta1
 kind: GrafanaContactPoint
@@ -18,7 +19,6 @@ spec:
     - type: webhook
       settings:
         httpMethod: POST
-        authorization_scheme: Bearer
         title: |-
           {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
       valuesFrom:
@@ -27,11 +27,6 @@ spec:
             secretKeyRef:
               name: rootly-alerts-o11y
               key: url
-        - targetPath: authorization_credentials
-          valueFrom:
-            secretKeyRef:
-              name: rootly-alerts-o11y
-              key: token
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
 apiVersion: grafana.integreatly.org/v1beta1
@@ -48,7 +43,6 @@ spec:
     - type: webhook
       settings:
         httpMethod: POST
-        authorization_scheme: Bearer
         title: |-
           {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
       valuesFrom:
@@ -57,11 +51,6 @@ spec:
             secretKeyRef:
               name: rootly-alerts-yucca
               key: url
-        - targetPath: authorization_credentials
-          valueFrom:
-            secretKeyRef:
-              name: rootly-alerts-yucca
-              key: token
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
 apiVersion: grafana.integreatly.org/v1beta1
@@ -78,7 +67,6 @@ spec:
     - type: webhook
       settings:
         httpMethod: POST
-        authorization_scheme: Bearer
         title: |-
           {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
       valuesFrom:
@@ -87,11 +75,6 @@ spec:
             secretKeyRef:
               name: rootly-alerts-fmeet
               key: url
-        - targetPath: authorization_credentials
-          valueFrom:
-            secretKeyRef:
-              name: rootly-alerts-fmeet
-              key: token
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
 apiVersion: grafana.integreatly.org/v1beta1
@@ -108,7 +91,6 @@ spec:
     - type: webhook
       settings:
         httpMethod: POST
-        authorization_scheme: Bearer
         title: |-
           {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
       valuesFrom:
@@ -117,11 +99,6 @@ spec:
             secretKeyRef:
               name: rootly-alerts-harbor
               key: url
-        - targetPath: authorization_credentials
-          valueFrom:
-            secretKeyRef:
-              name: rootly-alerts-harbor
-              key: token
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
 apiVersion: grafana.integreatly.org/v1beta1
@@ -138,7 +115,6 @@ spec:
     - type: webhook
       settings:
         httpMethod: POST
-        authorization_scheme: Bearer
         title: |-
           {{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
       valuesFrom:
@@ -147,8 +123,3 @@ spec:
             secretKeyRef:
               name: rootly-alerts-fip
               key: url
-        - targetPath: authorization_credentials
-          valueFrom:
-            secretKeyRef:
-              name: rootly-alerts-fip
-              key: token

kubernetes/apps/base/grafana/app/externalsecret.yaml

@@ -85,9 +85,6 @@ spec:
     - secretKey: url
       remoteRef:
         key: ROOTLY_ALERTS_O11Y_URL
-    - secretKey: token
-      remoteRef:
-        key: ROOTLY_ALERTS_O11Y_SECRET
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
 apiVersion: external-secrets.io/v1
@@ -104,9 +101,6 @@ spec:
     - secretKey: url
       remoteRef:
         key: ROOTLY_ALERTS_YUCCA_URL
-    - secretKey: token
-      remoteRef:
-        key: ROOTLY_ALERTS_YUCCA_SECRET
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
 apiVersion: external-secrets.io/v1
@@ -123,9 +117,6 @@ spec:
     - secretKey: url
       remoteRef:
         key: ROOTLY_ALERTS_FMEET_URL
-    - secretKey: token
-      remoteRef:
-        key: ROOTLY_ALERTS_FMEET_SECRET
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
 apiVersion: external-secrets.io/v1
@@ -142,9 +133,6 @@ spec:
     - secretKey: url
       remoteRef:
         key: ROOTLY_ALERTS_HARBOR_URL
-    - secretKey: token
-      remoteRef:
-        key: ROOTLY_ALERTS_HARBOR_SECRET
 ---
 # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
 apiVersion: external-secrets.io/v1
@@ -161,6 +149,3 @@ spec:
     - secretKey: url
       remoteRef:
         key: ROOTLY_ALERTS_FIP_URL
-    - secretKey: token
-      remoteRef:
-        key: ROOTLY_ALERTS_FIP_SECRET

kubernetes/apps/base/grafana/app/notificationpolicy.yaml

@@ -35,9 +35,11 @@ spec:
       - receiver: rootly-fmeet
         object_matchers:
           - ["grafana_folder", "=", "fmeet"]
+      # The harbor bundle titles its folder "Harbor"; grafana_folder carries the
+      # title, so match it case-insensitively rather than by the CR name.
       - receiver: rootly-harbor
         object_matchers:
-          - ["grafana_folder", "=", "harbor"]
+          - ["grafana_folder", "=~", "(?i)^harbor$"]
       - receiver: rootly-fip
         object_matchers:
           - ["grafana_folder", "=", "fip"]