feat(ceph): deploy monk onto the mon hosts (#588)

* feat(ceph): deploy monk onto the mon hosts

* fix(ceph): target the ceph_mon group and no_log the keyring tasks

* chore(ceph): drop the monk interval vars, monk reads the cluster now

* fix(ceph): restart monk when the pull lands a new digest

ansible/ceph/.mise.toml

@@ -104,6 +104,10 @@ run = "scripts/ansible-play.sh drift.yml"
 description = "Ship ceph node journald to o11y over the overlay (fluent-bit)"
 run = "scripts/ansible-play.sh logging.yml"
 
+[tasks.monk]
+description = "Deploy monk, the scrub-backlog exporter, on the mon hosts"
+run = "scripts/ansible-play.sh monk.yml"
+
 [tasks.netbird]
 description = "Enroll ceph nodes into NetBird (reads the setup key from 1Password)"
 run = """

ansible/ceph/monk.yml

@@ -0,0 +1,26 @@
+---
+# Run monk (the measured scrub-backlog exporter, packages/monk) on the
+# mon-role hosts. The mgr prometheus module exports no per-PG scrub stamps, so
+# the scrub dashboard's backlog numbers are estimates until monk serves the
+# measured ones on :9284 over the fabric, next to the other exporters vmagent
+# scrapes.
+#
+# Not on the Ceph critical path; runs after harden.yml so the unit starts
+# against the final nftables ruleset, and alongside logging.yml in spirit: a
+# node that cannot export scrub metrics must never block convergence.
+#
+# ceph_mon, not ceph_nodes: the TF-rendered group holds the hosts whose roles
+# include "mon" on any cluster shape (all of sietch, 5 of spice's 48). A
+# per-host mon var only exists on clusters that pin their quorum.
+#
+#   Canary:   mise run monk -- --limit spice-ceph-adelia
+#   Fleet:    mise run monk
+- name: Scrub-backlog exporter
+  hosts: ceph_mon
+  become: true
+  max_fail_percentage: 20
+  tasks:
+    - name: Deploy monk via the scrub_exporter role
+      ansible.builtin.import_role:
+        name: scrub_exporter
+      when: ceph_scrub_exporter_enabled | default(false) | bool

ansible/ceph/roles/scrub_exporter/defaults/main.yml

@@ -0,0 +1,16 @@
+---
+ceph_scrub_exporter_enabled: false
+
+# Delivered by yucca CI (deploy.yml pushes ghcr.io/immich-app/yucca/monk on
+# merge to main). No tag exists until the first release after the monk package
+# lands, so there is no usable default: pin a v<version> tag per cluster and
+# bump it deliberately, like the cluster ceph image.
+ceph_scrub_exporter_image: ""
+
+ceph_scrub_exporter_port: 9284
+ceph_scrub_exporter_refresh: 2m
+
+# Read-only identity; the keyring lands only on mon-role hosts. The container
+# runs as the ceph image's uid 167, which must be able to read it.
+ceph_scrub_exporter_auth_entity: client.scrub-exporter
+ceph_scrub_exporter_config_dir: /etc/ceph/monk

ansible/ceph/roles/scrub_exporter/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart monk
+  ansible.builtin.systemd:
+    name: monk
+    state: restarted
+    daemon_reload: true

ansible/ceph/roles/scrub_exporter/tasks/main.yml

@@ -0,0 +1,88 @@
+---
+- name: Require an image pin when the exporter is enabled
+  ansible.builtin.assert:
+    that:
+      - ceph_scrub_exporter_image | length > 0
+    fail_msg: >-
+      ceph_scrub_exporter_enabled is true but ceph_scrub_exporter_image is
+      empty; pin a released ghcr.io/immich-app/yucca/monk tag.
+  run_once: true
+
+# Cluster-level reads run once on the bootstrap host (the only host holding
+# admin creds) and register for every host in the play.
+- name: Create the read-only auth identity
+  ansible.builtin.command:
+    argv:
+      - ceph
+      - auth
+      - get-or-create
+      - "{{ ceph_scrub_exporter_auth_entity }}"
+      - mon
+      - allow r
+      - mgr
+      - allow r
+  register: scrub_exporter_keyring
+  changed_when: false
+  no_log: true
+  run_once: true
+  delegate_to: "{{ groups['ceph_bootstrap'] | first }}"
+
+- name: Render the minimal cluster config
+  ansible.builtin.command: ceph config generate-minimal-conf
+  register: scrub_exporter_conf
+  changed_when: false
+  run_once: true
+  delegate_to: "{{ groups['ceph_bootstrap'] | first }}"
+
+- name: Create the config directory
+  ansible.builtin.file:
+    path: "{{ ceph_scrub_exporter_config_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+# The trailing newline is load-bearing: ceph's conf_read_file rejects a file
+# without one, and ansible's .stdout strips it.
+- name: Stage ceph.conf
+  ansible.builtin.copy:
+    content: "{{ scrub_exporter_conf.stdout }}\n"
+    dest: "{{ ceph_scrub_exporter_config_dir }}/ceph.conf"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart monk
+
+# uid 167 = the ceph user inside the image; monk runs as it, not root, and a
+# root-owned 0600 keyring fails as "no keyring found".
+- name: Stage the keyring
+  ansible.builtin.copy:
+    content: "{{ scrub_exporter_keyring.stdout }}\n"
+    dest: "{{ ceph_scrub_exporter_config_dir }}/ceph.{{ ceph_scrub_exporter_auth_entity }}.keyring"
+    owner: "167"
+    group: "167"
+    mode: "0400"
+  no_log: true
+  notify: Restart monk
+
+- name: Pull the monk image
+  ansible.builtin.command: podman pull {{ ceph_scrub_exporter_image }}
+  register: scrub_exporter_pull
+  changed_when: "'Copying blob' in scrub_exporter_pull.stderr"
+  notify: Restart monk
+
+- name: Install the monk unit
+  ansible.builtin.template:
+    src: monk.service.j2
+    dest: /etc/systemd/system/monk.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart monk
+
+- name: Enable and start monk
+  ansible.builtin.systemd:
+    name: monk
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/ceph/roles/scrub_exporter/templates/monk.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=monk ceph scrub-backlog exporter
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+ExecStartPre=-/usr/bin/podman rm -f monk
+ExecStart=/usr/bin/podman run --rm --name monk --net host \
+  -v {{ ceph_scrub_exporter_config_dir }}:/etc/ceph:ro \
+  {{ ceph_scrub_exporter_image }} \
+  -listen {{ ceph_service_ip }}:{{ ceph_scrub_exporter_port }} \
+  -ceph-cmd "ceph --id {{ ceph_scrub_exporter_auth_entity | regex_replace('^client\\.', '') }}" \
+  -refresh {{ ceph_scrub_exporter_refresh }}
+ExecStop=/usr/bin/podman stop monk
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

ansible/ceph/roles/security/defaults/main.yml

@@ -42,6 +42,9 @@ ceph_firewall_alertmanager_port: "{{ ceph_alertmanager_port | default(9093) }}"
 ceph_firewall_node_exporter_port: 9100
 ceph_firewall_mgr_exporter_port: 9283
 ceph_firewall_ceph_exporter_port: 9926
+# monk, the scrub-backlog exporter (roles/scrub_exporter). Mon hosts only, but
+# opened everywhere like fluentbit: where the unit is absent nothing listens.
+ceph_firewall_scrub_exporter_port: 9284
 # Fluent Bit's own metrics (roles/fluentbit). Bound to the fabric public address
 # so vmagent reaches it the same way as the exporters above. Opening the port
 # where the agent is not enabled is inert: nothing listens.

ansible/ceph/roles/security/templates/nftables.conf.j2

@@ -69,6 +69,7 @@ table inet filter {
         ip saddr {{ net }} tcp dport {{ ceph_firewall_node_exporter_port }} accept
         ip saddr {{ net }} tcp dport {{ ceph_firewall_mgr_exporter_port }} accept
         ip saddr {{ net }} tcp dport {{ ceph_firewall_ceph_exporter_port }} accept
+        ip saddr {{ net }} tcp dport {{ ceph_firewall_scrub_exporter_port }} accept
         ip saddr {{ net }} tcp dport {{ ceph_firewall_fluentbit_port }} accept
         ip saddr {{ net }} tcp dport {{ ceph_firewall_service_discovery_port }} accept
 {% if ceph_firewall_iscsi_enabled | bool %}

ansible/ceph/site.yml

@@ -36,3 +36,6 @@
 
 - name: Host log shipping (journald to o11y; gated by ceph_fluentbit_enabled)
   import_playbook: logging.yml
+
+- name: Scrub-backlog exporter (monk on mon hosts; gated by ceph_scrub_exporter_enabled)
+  import_playbook: monk.yml