feat(ceph): netbird enrollment role for the spice nodes (#287)

ansible/ceph/.mise.toml

@@ -100,6 +100,21 @@ run = "scripts/ansible-play.sh status.yml"
 description = "Detect configuration drift from expected state"
 run = "scripts/ansible-play.sh drift.yml"
 
+[tasks.netbird]
+description = "Enroll ceph nodes into NetBird (reads the setup key from 1Password)"
+run = """
+#!/usr/bin/env bash
+set -euo pipefail
+# op creds: CI provides a SA token in OP_SERVICE_ACCOUNT_TOKEN; locally we use
+# the team-futo desktop session (same pattern as mgmt:ansible).
+ACCT=(); [ -z "${OP_SERVICE_ACCOUNT_TOKEN:-}" ] && ACCT=(--account "${OP_ACCOUNT:-team-futo}")
+# Only prod/htz-fsn1 mints a ceph setup key today (tf netbird stack). The key is
+# needed for FRESH enrollment only; converge of enrolled nodes runs key-free via
+# site.yml/deploy.
+NB_SETUP_KEY=$(op read "${ACCT[@]}" "op://yucca_tf_prod/NETBIRD_YUCCA_PROD_HTZ_FSN1_CEPH_SETUP_KEY/password")
+scripts/ansible-play.sh netbird.yml --extra-vars "ceph_netbird_setup_key=$NB_SETUP_KEY" "$@"
+"""
+
 [tasks.deploy]
 description = "Full deploy: baseline → tune → deploy → tune ceph → harden"
 run = """

ansible/ceph/inventories/prod-htz-fsn1/spice/group_vars/all/vars.yml

@@ -326,3 +326,10 @@ ceph_firewall_rgw_any_source: false
 ceph_firewall_trusted_ifaces:
   - bond0.122
   - wt0
+
+# === NetBird ===
+# Spice nodes are NetBird endpoint peers (group yucca-prod-htz-fsn1-ceph):
+# operator + CI SSH arrives over the overlay, governed by NetBird policies.
+# Fresh enrollment: `mise run netbird` (canary --limit spice-ceph-philip first);
+# converge of enrolled nodes is key-free. See roles/netbird.
+ceph_netbird_enabled: true

ansible/ceph/netbird.yml

@@ -0,0 +1,23 @@
+---
+# Enroll ceph nodes as NetBird overlay peers (endpoints only: no routes, no
+# mesh, no router role - replication and RGW stay on the fabric).
+#
+# Gated by ceph_netbird_enabled (spice: group_vars true; clusters without a
+# minted setup key leave it false). Fresh enrollment needs the setup key:
+#
+#   Canary:   mise run netbird -- --limit spice-ceph-philip
+#   Fleet:    mise run netbird
+#
+# Converging already-enrolled nodes is key-free, so site.yml carries this play
+# and the CI deploy pipeline keeps nodes converged with no secrets plumbing; a
+# reimaged (unenrolled) node fails with instructions instead of half-joining.
+- name: NetBird overlay enrollment
+  hosts: ceph_nodes
+  become: true
+  serial: "{{ netbird_serial | default(4) }}"
+  max_fail_percentage: 0
+  tasks:
+    - name: Enroll via the netbird role
+      ansible.builtin.import_role:
+        name: netbird
+      when: ceph_netbird_enabled | default(false) | bool

ansible/ceph/roles/netbird/defaults/main.yml

@@ -0,0 +1,24 @@
+---
+# NetBird enrollment defaults for ceph nodes. Endpoints only: no router role,
+# no IP forwarding, no route acceptance (the ceph peer group is kept out of
+# every network's distribution groups in TF, so the client receives zero
+# overlay routes and the fabric stays authoritative for cluster traffic).
+# Access to the node over the overlay is governed by NetBird policies
+# (yucca operators via the auto policy, CI via ci-to-ceph tcp/22); the node
+# firewall accepts wt0 wholesale (security role ceph_firewall_trusted_ifaces).
+ceph_netbird_enabled: false
+
+# Pinned + dpkg-held: netbird is load-bearing access infrastructure and its
+# ssh/DNAT behavior was audited against this version's source. Upgrades are
+# deliberate: bump the pin here (the install task unholds, installs, re-holds).
+ceph_netbird_version: "0.73.2"
+
+ceph_netbird_management_url: "https://api.netbird.io"
+
+# Setup key - NEVER committed. `mise run netbird` reads it from 1Password
+# (op://yucca_tf_prod/NETBIRD_YUCCA_PROD_HTZ_FSN1_CEPH_SETUP_KEY) and passes
+# it as an extra-var. Only FRESH enrollment needs it; converging an already
+# connected node skips `netbird up` entirely, so the CI deploy pipeline can
+# carry this role with no secrets plumbing. A reimaged node fails the assert
+# below with instructions until an operator runs `mise run netbird` for it.
+ceph_netbird_setup_key: ""

ansible/ceph/roles/netbird/tasks/main.yml

@@ -0,0 +1,120 @@
+---
+# Install NetBird (pinned) and join the overlay as an endpoint peer.
+# Convergeable: install is pin+hold, `netbird up` is skipped when connected.
+# Adapted from ansible/mgmt/roles/netbird, minus the route-peer parts (no IP
+# forwarding here - ceph nodes must never route or receive overlay routes).
+
+- name: Ensure keyrings directory exists
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: '0755'
+
+- name: Install the NetBird apt signing key
+  ansible.builtin.shell: |
+    set -o pipefail
+    curl -fsSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /etc/apt/keyrings/netbird.gpg
+  args:
+    creates: /etc/apt/keyrings/netbird.gpg
+    executable: /bin/bash
+
+- name: Add the NetBird apt repository
+  ansible.builtin.copy:
+    content: "deb [signed-by=/etc/apt/keyrings/netbird.gpg] https://pkgs.netbird.io/debian stable main\n"
+    dest: /etc/apt/sources.list.d/netbird.list
+    mode: '0644'
+
+- name: Update apt cache
+  # Always refresh: a cache_valid_time skip after a repo add installs against a
+  # stale index (same lesson as ceph_deploy/prerequisites).
+  ansible.builtin.apt:
+    update_cache: true
+  changed_when: false
+
+- name: Unhold netbird for the pinned install (no-op unless previously held)
+  ansible.builtin.dpkg_selections:
+    name: netbird
+    selection: install
+  failed_when: false
+
+- name: Install NetBird at the pinned version
+  ansible.builtin.apt:
+    name: "netbird={{ ceph_netbird_version }}"
+    state: present
+    allow_downgrade: true
+
+- name: Hold netbird at the pinned version
+  ansible.builtin.dpkg_selections:
+    name: netbird
+    selection: hold
+
+- name: Enable and start the netbird service
+  ansible.builtin.systemd:
+    name: netbird
+    enabled: true
+    state: started
+
+- name: Check NetBird connection status
+  ansible.builtin.command: netbird status
+  register: ceph_netbird_status
+  changed_when: false
+  failed_when: false
+
+- name: Assert the setup key is provided (fresh enrollment only)
+  ansible.builtin.assert:
+    that:
+      - ceph_netbird_setup_key | length > 0
+    fail_msg: >-
+      {{ inventory_hostname }} is not connected to NetBird and
+      ceph_netbird_setup_key is empty. Fresh enrollment needs the key: run
+      `mise run netbird` (reads NETBIRD_YUCCA_PROD_HTZ_FSN1_CEPH_SETUP_KEY
+      from 1Password), optionally with -- --limit {{ inventory_hostname }}.
+      Already-enrolled nodes converge without it.
+  when: "'Management: Connected' not in ceph_netbird_status.stdout"
+
+- name: Join the overlay
+  ansible.builtin.command:
+    cmd: >-
+      netbird up
+      --setup-key {{ ceph_netbird_setup_key }}
+      --management-url {{ ceph_netbird_management_url }}
+      --hostname {{ inventory_hostname }}
+  register: ceph_netbird_up
+  changed_when: ceph_netbird_up.rc == 0
+  no_log: true
+  when: "'Management: Connected' not in ceph_netbird_status.stdout"
+
+- name: Re-check status after join
+  ansible.builtin.command: netbird status
+  register: ceph_netbird_status_post
+  changed_when: false
+  retries: 6
+  delay: 5
+  until: "'Management: Connected' in ceph_netbird_status_post.stdout"
+  # Same condition as the join task: a node that was already connected never
+  # joined, so there is nothing to re-check (and _post stays undefined for the
+  # asserts' default() fallback).
+  when: "'Management: Connected' not in ceph_netbird_status.stdout"
+
+- name: Assert connected
+  ansible.builtin.assert:
+    that:
+      - >-
+        'Management: Connected'
+        in (ceph_netbird_status_post.stdout | default(ceph_netbird_status.stdout))
+    success_msg: "{{ inventory_hostname }}: NetBird connected"
+
+- name: Assert no overlay routes were received (fabric stays authoritative)
+  # `Networks: -` is the 0.73.x status line for zero received routes. If this
+  # fires, the ceph group leaked into a network's distribution groups in TF -
+  # an overlay route here could shadow the node's fabric paths. Format is
+  # version-coupled; revisit when bumping ceph_netbird_version.
+  ansible.builtin.assert:
+    that:
+      - >-
+        'Networks: -'
+        in (ceph_netbird_status_post.stdout | default(ceph_netbird_status.stdout))
+    fail_msg: >-
+      {{ inventory_hostname }} received overlay network routes - the ceph peer
+      group must stay out of all NetBird network distribution groups. Fix the
+      TF (tf/deployment/prod/htz-fsn1/netbird) before enrolling further nodes.

ansible/ceph/site.yml

@@ -16,6 +16,9 @@
 - name: OS baseline (ops user, packages, /etc/hosts)
   import_playbook: baseline.yml
 
+- name: NetBird overlay enrollment (endpoints only; gated by ceph_netbird_enabled)
+  import_playbook: netbird.yml
+
 - name: OS tuning (sysctl, TCP buffers)
   import_playbook: tune-os.yml