feat(ceph): route spice alertmanager to a real receiver (#355)

ansible/ceph/inventories/prod-htz-fsn1/spice/group_vars/all/vars.yml

@@ -45,6 +45,15 @@ ceph_service_ip: "{{ ceph_public_ip | default(bond_ip) }}"
 # every interface, unchanged); spice pins the fabric public network.
 ceph_bind_networks:
   - "{{ public_network }}"
+
+# Alert delivery. Without a receiver the ~89 Prometheus rules fire into
+# alertmanager's null route, which is how osd.35's failing disk and two degraded
+# bonds all went unnoticed until a scrub tripped HEALTH_ERR (2026-07-27). The URL
+# is a Zulip incoming webhook whose alertmanager integration accepts the standard
+# webhook payload; it carries an API key, so it lives in 1Password and arrives
+# through secrets.yml.tpl rather than being written here.
+ceph_alertmanager_webhook_urls:
+  - "{{ vault_alertmanager_webhook_url }}"
 # Bond of the two 25G NICs, LACP to match the leaf ae<k> aggregate (cluster-fabric).
 bond_mode: "802.3ad"
 # COEXISTENCE: keep the 1G WAN (oob_nic) on ifupdown exactly as installimage set

tf/deployment/prod/htz-fsn1/ceph/clusters.auto.tfvars

@@ -28,6 +28,9 @@ clusters = {
     ansible_ssh_key   = "~/.ssh/id_ed25519_spice"
     vault             = "yucca_tf_prod"
     provision_profile = null
+    # SPICE_CEPH_ALERTMANAGER_WEBHOOK_URL is provisioned out of band (Zulip
+    # incoming webhook) and referenced, never generated. See secrets.tf.
+    alertmanager_webhook = true
     hosts = [
       { name = "adelia", bond_ip = "178.63.139.248", bootstrap = true, roles = ["mon", "mgr", "osd", "rgw"] }, # srv 3008187 host_index 4  MON
       { name = "alexus", bond_ip = "178.63.139.254", roles = ["osd", "rgw"] },                                 # srv 3008189 host_index 5

tf/deployment/prod/htz-fsn1/ceph/main.tf

@@ -15,6 +15,8 @@ module "cluster" {
 
   provision_profile = each.value.provision_profile
 
+  alertmanager_webhook = each.value.alertmanager_webhook
+
   # Password items (<CLUSTER>_CEPH_*) are created at the stack level in
   # secrets.tf (mirrors tf/deployment/staging/talos), using this module's
   # `secrets` output for the item titles. The module itself stays free of the

tf/deployment/prod/htz-fsn1/ceph/secrets.tf

@@ -24,7 +24,11 @@
 locals {
   # Secret roles owned by the immich/restic contract, provisioned out-of-band.
   # Excluded from TF management here (see header).
-  ceph_unmanaged_secret_roles = ["s3_restic_access", "s3_restic_secret"]
+  # alertmanager_webhook holds an externally-issued receiver URL (Zulip incoming
+  # webhook), not a generated credential. Managing it here would overwrite the
+  # live URL with a random 32-char password on the next apply and silently break
+  # alert delivery, so TF only ever references the item.
+  ceph_unmanaged_secret_roles = ["s3_restic_access", "s3_restic_secret", "alertmanager_webhook"]
 
   # Per-role generated-password length. ops is the break-glass account typed by
   # hand at the KVM/console, so keep it short; dashboard/grafana are web logins

tf/deployment/prod/htz-fsn1/ceph/variables.tf

@@ -10,6 +10,10 @@ variable "clusters" {
     ansible_ssh_key   = string
     vault             = optional(string, "Yucca")
     provision_profile = optional(string)
+    # Reference an out-of-band <CLUSTER>_CEPH_ALERTMANAGER_WEBHOOK_URL item so
+    # the cluster's alertmanager gets a real receiver. See the ceph-cluster
+    # module's alertmanager_webhook variable.
+    alertmanager_webhook = optional(bool, false)
     hosts = list(object({
       name      = optional(string)
       bond_ip   = string

tf/shared/modules/ceph-cluster/main.tf

@@ -25,7 +25,7 @@ module "names" {
   names        = [for h in var.hosts : h.name]
 }
 
-# The wordlist + shuffle moved into node-names — preserve the existing shuffle state
+# The wordlist + shuffle moved into node-names; preserve the existing shuffle state
 # so host names don't re-randomize on this refactor.
 moved {
   from = random_shuffle.names
@@ -59,7 +59,7 @@ locals {
   # Every Ceph-project item grep-matches *_CEPH_* across all clusters.
   secret_prefix = "${upper(var.cluster_name)}_CEPH"
 
-  secrets = {
+  secrets = merge({
     ops              = "${local.secret_prefix}_OPS_PASSWORD"
     dashboard        = "${local.secret_prefix}_DASHBOARD_PASSWORD"
     grafana          = "${local.secret_prefix}_GRAFANA_PASSWORD"
@@ -70,5 +70,13 @@ locals {
     # 1P contract, which is named by cluster, not by the ceph subsystem.
     metrics_worker_access = "${upper(var.cluster_name)}_METRICS_WORKER_ACCESS_KEY"
     metrics_worker_secret = "${upper(var.cluster_name)}_METRICS_WORKER_SECRET_KEY"
-  }
+    },
+    # Alertmanager receiver URL. Opt-in per cluster, and provisioned OUT OF BAND:
+    # the value is an externally-issued webhook (Zulip/Opsgenie/etc), not a
+    # generated password, so the role is listed in the stack's
+    # ceph_unmanaged_secret_roles and TF only ever references it.
+    var.alertmanager_webhook ? {
+      alertmanager_webhook = "${local.secret_prefix}_ALERTMANAGER_WEBHOOK_URL"
+    } : {}
+  )
 }

tf/shared/modules/ceph-cluster/templates/secrets.yml.tpl.tftpl

@@ -1,5 +1,5 @@
 ---
-# TF-GENERATED — do not edit by hand. Source: tf/deployment/<partition>/<region>/ceph/
+# TF-GENERATED: do not edit by hand. Source: tf/deployment/<partition>/<region>/ceph/
 #
 # Consumed by scripts/ansible-play.sh at playbook time.
 # This file contains only 1Password secret references; op inject parses the
@@ -7,7 +7,7 @@
 # partial reference or unescaped double-curly-brace. Keep comments plain.
 # Non-secret variables belong in group_vars/all/vars.yml.
 #
-# Vault: ${vault} · Cluster: ${cluster_name}
+# Vault: ${vault} | Cluster: ${cluster_name}
 
 vault_ops_password: op://${vault}/${secrets.ops}/password
 vault_ceph_dashboard_password: op://${vault}/${secrets.dashboard}/password
@@ -16,3 +16,6 @@ vault_s3_restic_access_key: op://${vault}/${secrets.s3_restic_access}/password
 vault_s3_restic_secret_key: op://${vault}/${secrets.s3_restic_secret}/password
 vault_metrics_worker_access_key: op://${vault}/${secrets.metrics_worker_access}/password
 vault_metrics_worker_secret_key: op://${vault}/${secrets.metrics_worker_secret}/password
+%{ if contains(keys(secrets), "alertmanager_webhook") ~}
+vault_alertmanager_webhook_url: op://${vault}/${secrets.alertmanager_webhook}/password
+%{ endif ~}

tf/shared/modules/ceph-cluster/variables.tf

@@ -50,7 +50,7 @@ variable "hosts" {
     Ordered list of hosts in this cluster. First host is the bootstrap node unless a different host has bootstrap=true.
     Each host:
       - name:       (optional) short identifier. If null, TF picks from wordlist.
-                    Once deployed, do NOT change — drives hostname and all identity.
+                    Once deployed, do NOT change: it drives hostname and all identity.
       - bond_ip:    primary IP address ansible connects to
       - bootstrap:  (optional) true for the cephadm bootstrap node; exactly one per cluster
       - roles:      list of Ceph roles (mon, mgr, osd, rgw); informational
@@ -94,7 +94,13 @@ variable "provision_profile" {
 }
 
 variable "name_seed" {
-  description = "Optional seed to re-roll the cluster's wordlist auto-names. Do NOT bump once hosts are deployed — renames every auto-named host."
+  description = "Optional seed to re-roll the cluster's wordlist auto-names. Do NOT bump once hosts are deployed; it renames every auto-named host."
   type        = string
   default     = "v1"
 }
+
+variable "alertmanager_webhook" {
+  description = "Emit a vault_alertmanager_webhook_url reference in secrets.yml.tpl, pointing at <CLUSTER>_CEPH_ALERTMANAGER_WEBHOOK_URL. The item is provisioned OUT OF BAND (it holds an externally-issued receiver URL, not a generated password) and is listed in the stack's ceph_unmanaged_secret_roles so TF never overwrites it. Leave false for clusters with no alert receiver: the reference would otherwise break `op inject` on a vault that has no such item."
+  type        = bool
+  default     = false
+}