mirror of
https://github.com/immich-app/yucca-o11y.git
synced 2026-09-30 13:23:23 +08:00
fix(rootly): pass the Grafana source secret as a query parameter (#343)
This commit is contained in:
Antoine Lecompte
@@ -4,8 +4,9 @@
|
||||
# status-page items. The o11y folder reuses the service the heartbeat targets.
|
||||
#
|
||||
# Grafana posts each grouped notification to the source's /notify/Service/<id>
|
||||
# endpoint with the secret as a bearer token; the notification title becomes
|
||||
# the alert summary and commonLabels become alert labels.
|
||||
# endpoint with the source secret as a query parameter; the notification title
|
||||
# becomes the alert summary, commonLabels become alert labels, and a resolved
|
||||
# notification resolves the alert.
|
||||
locals {
|
||||
projects = toset(["o11y", "yucca", "fmeet", "harbor", "fip"])
|
||||
|
||||
|
||||
@@ -29,19 +29,14 @@ resource "onepassword_item" "heartbeat_ping_secret" {
|
||||
}
|
||||
|
||||
# Per-project Grafana alert source credentials, consumed by the rootly-alerts-*
|
||||
# ExternalSecrets that feed the rootly-<project> contact points.
|
||||
# ExternalSecrets that feed the rootly-<project> contact points. The secret
|
||||
# rides in the URL: Rootly's Grafana endpoint only reads it from the `secret`
|
||||
# query parameter and answers 404 "integration cannot be found" to a bearer
|
||||
# header, so the URL item is the whole credential.
|
||||
resource "onepassword_item" "grafana_alerts_url" {
|
||||
for_each = local.projects
|
||||
vault = data.onepassword_vault.env.uuid
|
||||
title = "ROOTLY_ALERTS_${upper(each.key)}_URL"
|
||||
category = "password"
|
||||
password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}"
|
||||
}
|
||||
|
||||
resource "onepassword_item" "grafana_alerts_secret" {
|
||||
for_each = local.projects
|
||||
vault = data.onepassword_vault.env.uuid
|
||||
title = "ROOTLY_ALERTS_${upper(each.key)}_SECRET"
|
||||
category = "password"
|
||||
password = rootly_alerts_source.grafana[each.key].secret
|
||||
password = "${local.grafana_webhooks_base}/notify/Service/${local.project_service_ids[each.key]}?secret=${rootly_alerts_source.grafana[each.key].secret}"
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ Cluster-generic boards (Kubernetes views and system, node exporter, vmagent, Cil
|
||||
|
||||
## Alerting
|
||||
|
||||
**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-<project>` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own bearer secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's URL and secret into the env 1Password vault (`ROOTLY_ALERTS_<PROJECT>_URL` / `_SECRET`); an ExternalSecret materializes them into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica.
|
||||
**Contact points.** A `GrafanaContactPoint` per destination. Alerts go to [Rootly](https://rootly.com): one webhook contact point per project (`rootly-<project>` in `base/grafana/app/contactpoint-rootly-alerts.yaml`), each posting to that project's Rootly alert source with the source's own secret, plus `rootly-heartbeat` for the dead man's switch. The alert sources, per-project services and their credentials are managed by the `deployment/modules/rootly/cluster` Terraform module, which writes each source's webhook URL into the env 1Password vault as `ROOTLY_ALERTS_<PROJECT>_URL`; the secret rides in that URL's `secret` query parameter because Rootly's Grafana endpoint reads it nowhere else. An ExternalSecret materializes it into the Secret the contact point reads via `receivers[].valuesFrom` - never in git. A resolved Grafana notification resolves the Rootly alert. Rootly derives alert urgency from the `severity` label (`critical` -> High, `warning` -> Medium, otherwise Low) and, until escalation policies exist, forwards fired and resolved alerts to Discord from its own cloud. Contact points live in the shared Grafana Postgres, so with the HA replica gossip cluster a firing alert notifies **once**, not once per replica.
|
||||
|
||||
**Rootly's Grafana integration.** Besides the alert sources, Rootly has an account-level Grafana integration (Integrations > Grafana) that takes this cluster's Grafana URL and an Admin service account token; it is what lets Rootly deep-link rules and snapshot dashboards into incidents. Rootly exposes no API or Terraform surface for it, so it is installed by hand once per env. The `deployment/modules/grafana/cluster` module owns the `rootly` service account and its token and writes the token to the env vault as `ROOTLY_GRAFANA_SERVICE_ACCOUNT_TOKEN`; paste that and `https://grafana.<env domain>` into Rootly. It is a separate module from `rootly/cluster` so the latter stays plannable while Grafana is down.
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
---
|
||||
# One webhook contact point per Grafana folder, each posting to that project's
|
||||
# Rootly alert source (its own secret) on the project's service. The
|
||||
# notification title becomes the Rootly alert summary, so it stays stable
|
||||
# between the firing and resolved notifications of the same group.
|
||||
# Rootly alert source on the project's service. The URL carries the source
|
||||
# secret as a query parameter (Rootly reads it nowhere else). The notification
|
||||
# title becomes the Rootly alert summary, so it stays stable between the
|
||||
# firing and resolved notifications of the same group.
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
|
||||
apiVersion: grafana.integreatly.org/v1beta1
|
||||
kind: GrafanaContactPoint
|
||||
@@ -18,7 +19,6 @@ spec:
|
||||
- type: webhook
|
||||
settings:
|
||||
httpMethod: POST
|
||||
authorization_scheme: Bearer
|
||||
title: |-
|
||||
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
|
||||
valuesFrom:
|
||||
@@ -27,11 +27,6 @@ spec:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-o11y
|
||||
key: url
|
||||
- targetPath: authorization_credentials
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-o11y
|
||||
key: token
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
|
||||
apiVersion: grafana.integreatly.org/v1beta1
|
||||
@@ -48,7 +43,6 @@ spec:
|
||||
- type: webhook
|
||||
settings:
|
||||
httpMethod: POST
|
||||
authorization_scheme: Bearer
|
||||
title: |-
|
||||
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
|
||||
valuesFrom:
|
||||
@@ -57,11 +51,6 @@ spec:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-yucca
|
||||
key: url
|
||||
- targetPath: authorization_credentials
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-yucca
|
||||
key: token
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
|
||||
apiVersion: grafana.integreatly.org/v1beta1
|
||||
@@ -78,7 +67,6 @@ spec:
|
||||
- type: webhook
|
||||
settings:
|
||||
httpMethod: POST
|
||||
authorization_scheme: Bearer
|
||||
title: |-
|
||||
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
|
||||
valuesFrom:
|
||||
@@ -87,11 +75,6 @@ spec:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-fmeet
|
||||
key: url
|
||||
- targetPath: authorization_credentials
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-fmeet
|
||||
key: token
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
|
||||
apiVersion: grafana.integreatly.org/v1beta1
|
||||
@@ -108,7 +91,6 @@ spec:
|
||||
- type: webhook
|
||||
settings:
|
||||
httpMethod: POST
|
||||
authorization_scheme: Bearer
|
||||
title: |-
|
||||
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
|
||||
valuesFrom:
|
||||
@@ -117,11 +99,6 @@ spec:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-harbor
|
||||
key: url
|
||||
- targetPath: authorization_credentials
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-harbor
|
||||
key: token
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
|
||||
apiVersion: grafana.integreatly.org/v1beta1
|
||||
@@ -138,7 +115,6 @@ spec:
|
||||
- type: webhook
|
||||
settings:
|
||||
httpMethod: POST
|
||||
authorization_scheme: Bearer
|
||||
title: |-
|
||||
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
|
||||
valuesFrom:
|
||||
@@ -147,8 +123,3 @@ spec:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-fip
|
||||
key: url
|
||||
- targetPath: authorization_credentials
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: rootly-alerts-fip
|
||||
key: token
|
||||
|
||||
@@ -85,9 +85,6 @@ spec:
|
||||
- secretKey: url
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_O11Y_URL
|
||||
- secretKey: token
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_O11Y_SECRET
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
|
||||
apiVersion: external-secrets.io/v1
|
||||
@@ -104,9 +101,6 @@ spec:
|
||||
- secretKey: url
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_YUCCA_URL
|
||||
- secretKey: token
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_YUCCA_SECRET
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
|
||||
apiVersion: external-secrets.io/v1
|
||||
@@ -123,9 +117,6 @@ spec:
|
||||
- secretKey: url
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_FMEET_URL
|
||||
- secretKey: token
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_FMEET_SECRET
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
|
||||
apiVersion: external-secrets.io/v1
|
||||
@@ -142,9 +133,6 @@ spec:
|
||||
- secretKey: url
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_HARBOR_URL
|
||||
- secretKey: token
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_HARBOR_SECRET
|
||||
---
|
||||
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
|
||||
apiVersion: external-secrets.io/v1
|
||||
@@ -161,6 +149,3 @@ spec:
|
||||
- secretKey: url
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_FIP_URL
|
||||
- secretKey: token
|
||||
remoteRef:
|
||||
key: ROOTLY_ALERTS_FIP_SECRET
|
||||
|
||||
@@ -35,9 +35,11 @@ spec:
|
||||
- receiver: rootly-fmeet
|
||||
object_matchers:
|
||||
- ["grafana_folder", "=", "fmeet"]
|
||||
# The harbor bundle titles its folder "Harbor"; grafana_folder carries the
|
||||
# title, so match it case-insensitively rather than by the CR name.
|
||||
- receiver: rootly-harbor
|
||||
object_matchers:
|
||||
- ["grafana_folder", "=", "harbor"]
|
||||
- ["grafana_folder", "=~", "(?i)^harbor$"]
|
||||
- receiver: rootly-fip
|
||||
object_matchers:
|
||||
- ["grafana_folder", "=", "fip"]
|
||||
|
||||
Reference in New Issue
Block a user