fix(tf/ceph): stop writing rendered inventories via local_file (#155)

local_file stored the destination path in shared remote state, derived from
get_repo_root(). git worktrees each resolve get_repo_root() to their own root,
so state became bound to whichever worktree last applied. Any apply from a
different checkout then force-replaced every rendered file and rebound state.

The module now emits inventory and secrets content as a render output instead
of local_file resources. ansible/ceph/scripts/render-inventories.sh reads that
output and writes the files using a path derived from its own location, so no
checkout-specific path ever enters state.

Also exclude painbox from the managed clusters (in active use by Zack); its
spec moves to clusters.example.tfvars and its 1Password items are left alone.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ansible/ceph/scripts/ansible-play.sh

@@ -18,7 +18,7 @@ set -euo pipefail
 
 if [ ! -f "$CEPH_ENV" ]; then
   echo "ansible-play.sh: inventory not found: $CEPH_ENV" >&2
-  echo "  Hint: has 'tofu apply' been run in tf/deployment/<env>/ceph/?" >&2
+  echo "  Hint: render it — 'terragrunt apply' in tf/deployment/<env>/ceph/, then scripts/render-inventories.sh <env>." >&2
   exit 1
 fi
 
@@ -27,7 +27,7 @@ TEMPLATE="$CEPH_ENV_DIR/secrets.yml.tpl"
 
 if [ ! -f "$TEMPLATE" ]; then
   echo "ansible-play.sh: secrets template not found: $TEMPLATE" >&2
-  echo "  Hint: has 'tofu apply' been run in tf/deployment/<env>/ceph/?" >&2
+  echo "  Hint: render it — 'terragrunt apply' in tf/deployment/<env>/ceph/, then scripts/render-inventories.sh <env>." >&2
   exit 1
 fi
 

ansible/ceph/scripts/render-inventories.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# render-inventories.sh — write the TF-rendered Ansible inventories + secrets
+# templates into THIS checkout, from the dev/ceph stack's `render` output.
+#
+# WHY a wrapper instead of tofu `local_file`: local_file records the destination
+# path in state. With a shared remote backend that path is checkout-specific, so
+# it coupled state to whichever git worktree last applied (get_repo_root()
+# resolves per-worktree) — every apply elsewhere then force-replaced the files
+# and rebound state. Reading the content from a TF *output* and writing it here,
+# with the path derived from THIS script's own location, keeps checkout-specific
+# paths out of shared state entirely.
+#
+# Prereq: `terragrunt apply` has been run for the stack (so the `render` output
+# reflects the current cluster spec). This script is read-only against state.
+#
+# Usage (from anywhere):
+#   ansible/ceph/scripts/render-inventories.sh [env]     # env defaults to dev
+set -euo pipefail
+
+ENVIRONMENT="${1:-dev}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ANSIBLE_CEPH="$(cd "$SCRIPT_DIR/.." && pwd)"   # ansible/ceph
+REPO_ROOT="$(cd "$ANSIBLE_CEPH/../.." && pwd)" # repo root of THIS checkout
+STACK_DIR="$REPO_ROOT/tf/deployment/${ENVIRONMENT}/ceph"
+INV_ROOT="$ANSIBLE_CEPH/inventories"
+
+[ -d "$STACK_DIR" ] || {
+  echo "render-inventories: stack not found: $STACK_DIR" >&2
+  exit 1
+}
+
+# The `render` output is non-sensitive (inventory text + op:// references — no
+# raw secrets). Reading state needs the S3 backend creds, so go through the
+# op-run wrapper which injects them from tf/.env.
+JSON="$(cd "$STACK_DIR" && OP_ENV_FILE="$REPO_ROOT/tf/.env" "$REPO_ROOT/tf/op-run.sh" terragrunt output -json render)"
+
+# JSON travels via env (RENDER_JSON); the heredoc owns python's stdin (the
+# program), so we can't also pipe the data in.
+RENDER_JSON="$JSON" python3 - "$INV_ROOT" <<'PY'
+import json, os, sys
+
+inv_root = sys.argv[1]
+data = json.loads(os.environ["RENDER_JSON"])
+for cluster, spec in data.items():
+    d = os.path.join(inv_root, spec["dirname"])
+    os.makedirs(d, exist_ok=True)
+    for fname, content in spec["files"].items():
+        p = os.path.join(d, fname)
+        with open(p, "w") as fh:
+            fh.write(content)
+        os.chmod(p, 0o644)
+        print(f"wrote {p}")
+PY
+
+echo "render-inventories: done (${ENVIRONMENT})."

tf/deployment/dev/ceph/clusters.auto.tfvars

@@ -1,5 +1,10 @@
-# Declarative cluster inventory. Adding a cluster = add an entry here + `tofu apply`.
-# ansible_project_root is injected by terragrunt from the repo root.
+# Declarative cluster inventory. Adding a cluster = add an entry here, run
+# `terragrunt apply`, then `ansible/ceph/scripts/render-inventories.sh dev`.
+#
+# painbox is intentionally NOT managed here right now: it is in active use by
+# Zack for other purposes, so this stack must not render or reconcile it. Its
+# spec is kept as a reference in clusters.example.tfvars (not auto-loaded). Its
+# 1Password items are left untouched.
 
 clusters = {
   sietch = {
@@ -18,22 +23,4 @@ clusters = {
       { name = "samara", bond_ip = "10.10.10.92" },
     ]
   }
-
-  painbox = {
-    domain           = "dev.hel.htz.futo.cloud"
-    environment      = "dev"
-    datacenter       = "hel"
-    provider_code    = "htz"
-    role_in_hostname = "ceph"
-    ansible_ssh_user = "root"
-    ansible_ssh_key  = "~/.ssh/id_ed25519_painbox"
-    vault            = "yucca_tf_dev"
-    # Painbox runs Ceph Tentacle on Bookworm (1× SX295, single-node).
-    # Reprovisioned end-to-end via Hetzner installimage; no provision_profile
-    # because Hetzner installimage handles partitioning + base OS install.
-    # Auto-picked wordlist name: "evelyn" → painbox-ceph-evelyn.
-    hosts = [
-      { bond_ip = "157.180.105.198", bootstrap = true },
-    ]
-  }
 }

tf/deployment/dev/ceph/clusters.example.tfvars

@@ -0,0 +1,27 @@
+# Example cluster spec, NOT auto-loaded (only *.auto.tfvars is). Kept as a
+# reference for clusters that exist but are not currently managed by this stack.
+#
+# painbox: a single-node Ceph Tentacle cluster on Bookworm (1x SX295) in
+# Hetzner Helsinki, reprovisioned end to end via Hetzner installimage (hence no
+# provision_profile; installimage handles partitioning + base OS). It is in
+# active use by Zack, so it is excluded from clusters.auto.tfvars. To bring it
+# back under management, move this entry into the clusters map there, run
+# `terragrunt apply`, then render. Its 1Password items already exist in
+# yucca_tf_dev and are left untouched either way.
+#
+# clusters = {
+#   painbox = {
+#     domain           = "dev.hel.htz.futo.cloud"
+#     environment      = "dev"
+#     datacenter       = "hel"
+#     provider_code    = "htz"
+#     role_in_hostname = "ceph"
+#     ansible_ssh_user = "root"
+#     ansible_ssh_key  = "~/.ssh/id_ed25519_painbox"
+#     vault            = "yucca_tf_dev"
+#     # Auto-picked wordlist name: "evelyn" -> painbox-ceph-evelyn.
+#     hosts = [
+#       { bond_ip = "157.180.105.198", bootstrap = true },
+#     ]
+#   }
+# }

tf/deployment/dev/ceph/main.tf

@@ -1,14 +1,3 @@
-locals {
-  # Directory naming: <cluster>-ceph.<env>.<dc>.<provider>
-  # Project-scoped ("ceph"), not hostname-role-scoped — mirrors the secret-prefix
-  # convention (<CLUSTER>_CEPH_*) so all ceph-project artifacts are grep-able
-  # under *-ceph.* regardless of whether hosts are named with ceph/osd/mon role.
-  inventory_dirs = {
-    for cname, c in var.clusters :
-    cname => "${var.ansible_project_root}/inventories/${cname}-ceph.${c.environment}.${c.datacenter}.${c.provider_code}"
-  }
-}
-
 module "cluster" {
   for_each = var.clusters
   source   = "../../../shared/modules/ceph-cluster"
@@ -24,11 +13,6 @@ module "cluster" {
   vault            = coalesce(each.value.vault, "Yucca")
   hosts            = each.value.hosts
 
-  ansible_inventory_path           = "${local.inventory_dirs[each.key]}/inventory.ini"
-  ansible_destroy_inventory_path   = "${local.inventory_dirs[each.key]}/inventory-destroy.ini"
-  ansible_provision_inventory_path = "${local.inventory_dirs[each.key]}/inventory-provision.ini"
-  ansible_secrets_template_path    = "${local.inventory_dirs[each.key]}/secrets.yml.tpl"
-
   provision_profile = each.value.provision_profile
 
   # NOTE: onepassword_item provisioning is dormant per ADR-009. Re-enable
@@ -44,9 +28,23 @@ output "cluster_summaries" {
       fqdn           = m.fqdn_cluster
       bootstrap_host = m.bootstrap_host.hostname_short
       host_count     = length(m.hosts)
-      inventory      = m.inventory_path
-      secrets_tpl    = m.secrets_template_path
+      inventory_dir  = m.inventory_dirname
       secrets        = m.secrets
     }
   }
 }
+
+# Consumed by ansible/ceph/scripts/render-inventories.sh: it reads this output
+# and writes each cluster's files under ansible/ceph/inventories/<dirname>/.
+# Rendered content lives in a TF OUTPUT (not in local_file resources), so the
+# shared remote state never records a checkout-specific filesystem path. See
+# the module's rendering.tf for the full rationale.
+output "render" {
+  description = "Per-cluster { dirname, files } for the local render wrapper."
+  value = {
+    for k, m in module.cluster : k => {
+      dirname = m.inventory_dirname
+      files   = m.rendered_files
+    }
+  }
+}

tf/deployment/dev/ceph/terragrunt.hcl

@@ -3,9 +3,7 @@ include "root" {
 }
 
 # clusters.auto.tfvars is automatically loaded by OpenTofu in this directory.
-# No explicit inputs {} block needed for cluster declarations.
-inputs = {
-  # ansible_project_root is relative to the repo root; terragrunt resolves
-  # it via get_repo_root() so it works no matter where apply is run from.
-  ansible_project_root = "${get_repo_root()}/ansible/ceph"
-}
+# No inputs are injected here: rendered inventories are no longer written by
+# tofu (which previously needed get_repo_root() to locate ansible/ceph and so
+# coupled shared state to the applying worktree). Content is emitted via the
+# `render` output and written locally by ansible/ceph/scripts/render-inventories.sh.

tf/deployment/dev/ceph/variables.tf

@@ -1,8 +1,3 @@
-variable "ansible_project_root" {
-  description = "Absolute path to yucca/ansible/ceph/ where inventories/ live. Rendered inventories go under here."
-  type        = string
-}
-
 variable "clusters" {
   description = "Map of cluster spec keyed by short cluster name (sietch, painbox, ...)."
   type = map(object({

tf/shared/modules/ceph-cluster/outputs.tf

@@ -26,10 +26,12 @@ output "secrets" {
   value       = local.secrets
 }
 
-output "inventory_path" {
-  value = local_file.inventory.filename
+output "inventory_dirname" {
+  description = "Inventory directory name (<cluster>-ceph.<env>.<dc>.<provider>). The render wrapper writes files under ansible/ceph/inventories/<this>/."
+  value       = local.inventory_dirname
 }
 
-output "secrets_template_path" {
-  value = local_file.secrets_template.filename
+output "rendered_files" {
+  description = "Map of filename => rendered content (inventory.ini, inventory-destroy.ini, secrets.yml.tpl, plus inventory-provision.ini when a provision_profile is set). Written locally by scripts/render-inventories.sh — intentionally NOT a local_file, so no filesystem path enters shared state."
+  value       = local.rendered_files
 }

tf/shared/modules/ceph-cluster/rendering.tf

@@ -1,12 +1,27 @@
-# Rendered artifacts consumed by Ansible.
+# Rendered Ansible artifacts (inventory files + secrets template).
 #
-# These files are TF-generated — they must be in .gitignore on the Ansible side.
-# Re-run `terragrunt apply` (or `tofu apply`) after changing cluster spec.
+# These are exposed as module OUTPUTS (`rendered_files` / `inventory_dirname`),
+# NOT written by tofu via `local_file`. A local wrapper —
+# ansible/ceph/scripts/render-inventories.sh — reads the output and writes the
+# files into the operator's own checkout.
+#
+# WHY: `local_file` stores the destination filename in state. With a shared
+# remote backend that path is machine/checkout-specific, so it coupled the
+# state to whichever git worktree last ran apply (get_repo_root() resolves to
+# the *worktree* root). Any apply from a different checkout then saw the
+# filename change and force-replaced every file — destroying the previous
+# worktree's rendered files and rebinding shared state to the new path.
+# Keeping rendered content in outputs (and writing it locally) removes
+# filesystem paths from shared state entirely.
 
-resource "local_file" "inventory" {
-  filename        = var.ansible_inventory_path
-  file_permission = "0644"
-  content = templatefile("${path.module}/templates/inventory.ini.tftpl", {
+locals {
+  # Inventory directory name: <cluster>-ceph.<env>.<dc>.<provider>.
+  # Project-scoped ("ceph"), not hostname-role-scoped — mirrors the secret
+  # prefix (<CLUSTER>_CEPH_*) so all ceph-project artifacts are grep-able under
+  # *-ceph.* regardless of whether hosts are named with a ceph/osd/mon role.
+  inventory_dirname = "${var.cluster_name}-ceph.${var.environment}.${var.datacenter}.${var.provider_code}"
+
+  _inventory_template_vars = {
     cluster_name     = var.cluster_name
     domain           = var.domain
     hosts            = local.hosts_computed
@@ -14,41 +29,27 @@ resource "local_file" "inventory" {
     join_hosts       = local.join_hosts
     ansible_ssh_user = var.ansible_ssh_user
     ansible_ssh_key  = var.ansible_ssh_key
-  })
-}
+  }
 
-resource "local_file" "destroy_inventory" {
-  filename        = var.ansible_destroy_inventory_path
-  file_permission = "0644"
-  content = templatefile("${path.module}/templates/inventory-destroy.ini.tftpl", {
-    cluster_name     = var.cluster_name
-    domain           = var.domain
-    hosts            = local.hosts_computed
-    bootstrap_host   = local.bootstrap_host
-    join_hosts       = local.join_hosts
-    ansible_ssh_user = var.ansible_ssh_user
-    ansible_ssh_key  = var.ansible_ssh_key
-  })
-}
-
-resource "local_file" "provision_inventory" {
-  count           = var.provision_profile == null ? 0 : 1
-  filename        = var.ansible_provision_inventory_path
-  file_permission = "0644"
-  content = templatefile("${path.module}/templates/inventory-provision-${var.provision_profile}.ini.tftpl", {
-    cluster_name = var.cluster_name
-    domain       = var.domain
-    hosts        = local.hosts_computed
-  })
-}
-
-resource "local_file" "secrets_template" {
-  filename        = var.ansible_secrets_template_path
-  file_permission = "0644"
-  content = templatefile("${path.module}/templates/secrets.yml.tpl.tftpl", {
-    cluster_name  = var.cluster_name
-    vault         = var.vault
-    secret_prefix = local.secret_prefix
-    secrets       = local.secrets
-  })
+  # filename => rendered content. inventory-provision.ini is only produced when
+  # a provision_profile is set (e.g., sietch's debian-live; painbox is null).
+  rendered_files = merge(
+    {
+      "inventory.ini"         = templatefile("${path.module}/templates/inventory.ini.tftpl", local._inventory_template_vars)
+      "inventory-destroy.ini" = templatefile("${path.module}/templates/inventory-destroy.ini.tftpl", local._inventory_template_vars)
+      "secrets.yml.tpl" = templatefile("${path.module}/templates/secrets.yml.tpl.tftpl", {
+        cluster_name  = var.cluster_name
+        vault         = var.vault
+        secret_prefix = local.secret_prefix
+        secrets       = local.secrets
+      })
+    },
+    var.provision_profile == null ? {} : {
+      "inventory-provision.ini" = templatefile("${path.module}/templates/inventory-provision-${var.provision_profile}.ini.tftpl", {
+        cluster_name = var.cluster_name
+        domain       = var.domain
+        hosts        = local.hosts_computed
+      })
+    }
+  )
 }

tf/shared/modules/ceph-cluster/variables.tf

@@ -72,25 +72,10 @@ variable "hosts" {
   }
 }
 
-variable "ansible_inventory_path" {
-  description = "Absolute path to write the rendered inventory.ini."
-  type        = string
-}
-
-variable "ansible_destroy_inventory_path" {
-  description = "Absolute path to write the rendered inventory-destroy.ini."
-  type        = string
-}
-
-variable "ansible_provision_inventory_path" {
-  description = "Absolute path to write inventory-provision.ini. Only rendered when provision_profile is non-null."
-  type        = string
-}
-
-variable "ansible_secrets_template_path" {
-  description = "Absolute path to write the rendered secrets.yml.tpl (consumed by op inject at playbook time)."
-  type        = string
-}
+# Rendered-artifact destination paths used to live here (ansible_*_path). They
+# were removed: rendering no longer writes files via local_file (which leaked a
+# checkout-specific path into shared state). The module now only emits content
+# via the `rendered_files` output; scripts/render-inventories.sh writes it.
 
 variable "provision_profile" {
   description = <<-EOT