feat(netbird-ansible): better subnet routers (#217)

tf/deployment/prod/htz-fsn1/mgmt-hosts.yaml

@@ -2,13 +2,19 @@
 # BOTH the reprovision stack (mgmt.tf, uses server_number) and the ansible
 # inventory render (tf/render/ansible-mgmt, uses everything else).
 #
-#   site_id        — drives addressing (must match the stack's var.site_id).
-#   cluster_id     — the cluster whose public/private VLANs these hosts sit on.
-#   host_index     — host's offset within each VLAN /23 (gateway is .1 on the leaf);
-#                    .2/.3 here -> 10.40.20.2/.3 (public), 10.40.22.2/.3 (private).
-#   fabric_nic     — 25G NIC carrying the tagged VLAN sub-interfaces (verify after
-#                    reprovision; predictable name may differ on fresh Debian 13).
-#   subnet_router  — advertises the mgmt /24 over Tailscale (exactly one host).
+#   site_id      — drives addressing (must match the stack's var.site_id).
+#   cluster_id   — the cluster whose public/private VLANs these hosts sit on.
+#   host_index   — host's offset within each fabric VLAN (gateway is .1 on the leaf);
+#                  .2/.3 here -> 10.40.20.2/.3 (public), 10.40.22.2/.3 (private),
+#                  10.40.10.2/.3 (api).
+#   fabric_nic   — 25G NIC carrying the tagged cluster/api VLAN sub-interfaces.
+#   oob_nic      — 1G NIC on the OOB management LAN (10.40.5.0/24, the switch vme).
+#   oob_host     — host octet on the OOB LAN (mgmt-1 .50, mgmt-2 .51).
+#
+# These hosts are the NetBird route peers for the site (see prod/htz-fsn1/netbird);
+# the L3 paths configured from this roster (ansible networkd role) are what let
+# them actually forward to the routed subnets. VERIFY both NIC names with
+# `ip link` after a reinstall — predictable names can differ on fresh Debian 13.
 site_id: 40
 cluster_id: 1
 
@@ -18,10 +24,12 @@ hosts:
     public_ip: 178.63.124.40
     host_index: 2
     fabric_nic: enp33s0f0np0
-    subnet_router: true
+    oob_nic: enp37s0
+    oob_host: 50
   htz-fsn-mgmt-2:
     server_number: 3008209
     public_ip: 178.63.124.41
     host_index: 3
     fabric_nic: enp33s0f0np0
-    subnet_router: false
+    oob_nic: enp37s0
+    oob_host: 51

tf/render/ansible-mgmt/main.tf

@@ -1,7 +1,7 @@
 # Render the ansible/mgmt inventory for a site from Terraform's sources of truth:
 #   • fabric-addressing — VLAN ids + per-host VLAN addresses
 #   • identity          — server login users (+ keys + sudo)
-#   • mgmt-hosts.yaml   — the host roster (IPs, NIC, host index, subnet router)
+#   • mgmt-hosts.yaml   — the host roster (IPs, NICs, host index, OOB octet)
 #
 # Lightweight by design (local provider only, no backend/secrets), so it can run
 # just-in-time in the ansible CI job (and locally) via `mise run mgmt:ansible`.
@@ -14,6 +14,8 @@ locals {
 
   pub_mask  = split("/", module.addressing.public_cidr)[1]
   priv_mask = split("/", module.addressing.private_cidr)[1]
+  api_mask  = split("/", module.addressing.api_cidr)[1]
+  mgmt_mask = split("/", module.addressing.mgmt_cidr)[1]
 
   header = "# GENERATED by `mise run mgmt:render-inventory` (tf/render/ansible-mgmt).\n# Do not edit — edit the Terraform sources (mgmt-hosts.yaml, fabric-addressing, identity).\n"
 }
@@ -42,20 +44,25 @@ resource "local_file" "hosts" {
   })}"
 }
 
-# Per-host: 25G NIC + the tagged VLAN sub-interfaces (ids + addresses from the
-# addressing module) + the subnet-router's advertised route.
+# Per-host L3 so the node can forward the NetBird-routed site subnets:
+#   • OOB 1G NIC on the management LAN (10.40.5.0/24 — the switch vme)
+#   • tagged VLAN sub-interfaces on the 25G NIC (public/private/api)
+# Addresses/ids come from the addressing module; bootstrap is the public IP
+# (site.yml reconnects over NetBird once the host joins).
 resource "local_file" "host_vars" {
   for_each = local.hosts
   filename = "${local.inv_dir}/host_vars/${each.key}.yml"
   content = "${local.header}${yamlencode({
-    # Bootstrap address — site.yml reconnects over NetBird once the host joins.
-    # (NetBird routes the site subnets via the mgmt peer group; that's declared in
-    # tf/deployment/prod/<site>/netbird, so there's no per-host advertise flag.)
-    mgmt_public_ip  = each.value.public_ip
+    mgmt_public_ip = each.value.public_ip
+    mgmt_oob = {
+      nic     = each.value.oob_nic
+      address = "${cidrhost(module.addressing.mgmt_cidr, each.value.oob_host)}/${local.mgmt_mask}"
+    }
     mgmt_fabric_nic = each.value.fabric_nic
     mgmt_fabric_vlans = [
       { id = module.addressing.public_vlan_id, address = "${cidrhost(module.addressing.public_cidr, each.value.host_index)}/${local.pub_mask}" },
       { id = module.addressing.private_vlan_id, address = "${cidrhost(module.addressing.private_cidr, each.value.host_index)}/${local.priv_mask}" },
+      { id = module.addressing.api_vlan_id, address = "${cidrhost(module.addressing.api_cidr, each.value.host_index)}/${local.api_mask}" },
     ]
   })}"
 }