immich-app/yucca-o11y
1
0
Fork 0
mirror of https://github.com/immich-app/yucca-o11y.git synced 2026-09-30 13:23:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(rootly): route Grafana alerts to Rootly per project (#342)

Browse Source
This commit is contained in:
Antoine Lecompte
2026-09-21 09:12:38 -04:00
committed by GitHub
parent 4a90d0c369
commit 31bdedf6ad
20 changed files with 568 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kubernetes/apps/base/grafana/app/contactpoint-discord.yaml
-31
View File
@@ -1,31 +0,0 @@
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: discord
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: discord
receivers:
- type: discord
settings:
use_embed_description: true
title: |-
{{ if .Alerts.Firing }}🔴 {{ .Alerts.Firing | len }} firing{{ else }}✅ resolved{{ end }} · {{ .CommonLabels.grafana_folder }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
message: |-
{{ range .Alerts -}}
**{{ .Labels.alertname }}**{{ if and .Labels.severity (ne .Labels.severity "none") }} `{{ .Labels.severity }}`{{ end }}
{{ if .Annotations.summary }}> {{ .Annotations.summary }}{{ if match "67" .Annotations.summary }} <a:pepe67:1500112466921652265>{{ end }}
{{ end }}[View rule]({{ .GeneratorURL }}){{ if .SilenceURL }} · [Silence]({{ .SilenceURL }}){{ end }}
{{ end -}}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: grafana-discord-webhook
key: url
kubernetes/apps/base/grafana/app/contactpoint-rootly-alerts.yaml
+154
View File
@@ -0,0 +1,154 @@
---
# One webhook contact point per Grafana folder, each posting to that project's
# Rootly alert source (its own secret) on the project's service. The
# notification title becomes the Rootly alert summary, so it stays stable
# between the firing and resolved notifications of the same group.
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: rootly-o11y
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: rootly-o11y
receivers:
- type: webhook
settings:
httpMethod: POST
authorization_scheme: Bearer
title: |-
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: rootly-alerts-o11y
key: url
- targetPath: authorization_credentials
valueFrom:
secretKeyRef:
name: rootly-alerts-o11y
key: token
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: rootly-yucca
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: rootly-yucca
receivers:
- type: webhook
settings:
httpMethod: POST
authorization_scheme: Bearer
title: |-
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: rootly-alerts-yucca
key: url
- targetPath: authorization_credentials
valueFrom:
secretKeyRef:
name: rootly-alerts-yucca
key: token
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: rootly-fmeet
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: rootly-fmeet
receivers:
- type: webhook
settings:
httpMethod: POST
authorization_scheme: Bearer
title: |-
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: rootly-alerts-fmeet
key: url
- targetPath: authorization_credentials
valueFrom:
secretKeyRef:
name: rootly-alerts-fmeet
key: token
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: rootly-harbor
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: rootly-harbor
receivers:
- type: webhook
settings:
httpMethod: POST
authorization_scheme: Bearer
title: |-
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: rootly-alerts-harbor
key: url
- targetPath: authorization_credentials
valueFrom:
secretKeyRef:
name: rootly-alerts-harbor
key: token
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/grafana.integreatly.org/grafanacontactpoint_v1beta1.json
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
name: rootly-fip
spec:
resyncPeriod: 1h
instanceSelector:
matchLabels:
dashboards: grafana
name: rootly-fip
receivers:
- type: webhook
settings:
httpMethod: POST
authorization_scheme: Bearer
title: |-
{{ .CommonLabels.grafana_folder }} · {{ .CommonLabels.alertname }}{{ if .CommonLabels.cluster }} · {{ .CommonLabels.cluster }}{{ end }}
valuesFrom:
- targetPath: url
valueFrom:
secretKeyRef:
name: rootly-alerts-fip
key: url
- targetPath: authorization_credentials
valueFrom:
secretKeyRef:
name: rootly-alerts-fip
key: token
kubernetes/apps/base/grafana/app/externalsecret.yaml
+85 -8
View File
@@ -74,16 +74,93 @@ spec:
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: grafana-discord-webhook
name: rootly-alerts-o11y
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-environment
target:
name: grafana-discord-webhook
template:
data:
url: "{{ .password }}"
dataFrom:
- extract:
key: GRAFANA_DISCORD_WEBHOOK
name: rootly-alerts-o11y
data:
- secretKey: url
remoteRef:
key: ROOTLY_ALERTS_O11Y_URL
- secretKey: token
remoteRef:
key: ROOTLY_ALERTS_O11Y_SECRET
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: rootly-alerts-yucca
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-environment
target:
name: rootly-alerts-yucca
data:
- secretKey: url
remoteRef:
key: ROOTLY_ALERTS_YUCCA_URL
- secretKey: token
remoteRef:
key: ROOTLY_ALERTS_YUCCA_SECRET
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: rootly-alerts-fmeet
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-environment
target:
name: rootly-alerts-fmeet
data:
- secretKey: url
remoteRef:
key: ROOTLY_ALERTS_FMEET_URL
- secretKey: token
remoteRef:
key: ROOTLY_ALERTS_FMEET_SECRET
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: rootly-alerts-harbor
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-environment
target:
name: rootly-alerts-harbor
data:
- secretKey: url
remoteRef:
key: ROOTLY_ALERTS_HARBOR_URL
- secretKey: token
remoteRef:
key: ROOTLY_ALERTS_HARBOR_SECRET
---
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/external-secrets.io/externalsecret_v1.json
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: rootly-alerts-fip
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-environment
target:
name: rootly-alerts-fip
data:
- secretKey: url
remoteRef:
key: ROOTLY_ALERTS_FIP_URL
- secretKey: token
remoteRef:
key: ROOTLY_ALERTS_FIP_SECRET
kubernetes/apps/base/grafana/app/kustomization.yaml
+1 -1
View File
@@ -8,8 +8,8 @@ resources:
- ./datasource-victoria-logs-legacy.yaml
- ./datasource-fleet.yaml
- ./datasource-victoriametrics.yaml
- ./contactpoint-discord.yaml
- ./contactpoint-rootly.yaml
- ./contactpoint-rootly-alerts.yaml
- ./notificationpolicy.yaml
- ./servicemonitor.yaml
- ./alerts-o11y.yaml
kubernetes/apps/base/grafana/app/notificationpolicy.yaml
+9 -8
View File
@@ -10,7 +10,8 @@ spec:
matchLabels:
dashboards: grafana
route:
receiver: discord
# Folders without a Rootly alert source of their own land on the o11y one.
receiver: rootly-o11y
group_by:
- grafana_folder
- alertname
@@ -25,18 +26,18 @@ spec:
group_wait: 15s
group_interval: 2m
repeat_interval: 2m
- receiver: discord
object_matchers:
- ["grafana_folder", "=", "yucca"]
- receiver: discord
- receiver: rootly-o11y
object_matchers:
- ["grafana_folder", "=", "o11y"]
- receiver: discord
- receiver: rootly-yucca
object_matchers:
- ["grafana_folder", "=", "yucca"]
- receiver: rootly-fmeet
object_matchers:
- ["grafana_folder", "=", "fmeet"]
- receiver: discord
- receiver: rootly-harbor
object_matchers:
- ["grafana_folder", "=", "harbor"]
- receiver: discord
- receiver: rootly-fip
object_matchers:
- ["grafana_folder", "=", "fip"]