feat: local k8s (#85)

* impl. local kube

* add support for op injected oidc secrets

* ci: set least-privilege workflow token permissions
This commit is contained in:
Antoine Lecompte
2026-06-12 13:17:37 +00:00
committed by GitHub
parent 179ae89605
commit 070e22a7bb
122 changed files with 2814 additions and 52 deletions
+9
View File
@@ -11,3 +11,12 @@ e2e
**/.env*
Dockerfile
packages/**/build
.dev
.mise/data
**/tmp
**/.turbo
**/.cache
charts
Tiltfile
k3d.yaml
compose.yml
+71 -27
View File
@@ -5,6 +5,11 @@ on:
push:
branches: [main]
# Every job only reads the repo (checkout + tool downloads); nothing writes
# back through the token.
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
@@ -20,14 +25,19 @@ jobs:
- name: Setup Mise
uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0
env:
# mise downloads tools from GitHub releases; anonymous requests hit
# API rate limits with four parallel jobs.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: mise prepare
- name: Run checks
run: mise check
integration:
name: Integration Tests
k8s-validate:
name: Validate Kubernetes Surface
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
@@ -35,17 +45,53 @@ jobs:
- name: Setup Mise
uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0
env:
# mise downloads tools from GitHub releases; anonymous requests hit
# API rate limits with four parallel jobs.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# helm template + kubeconform per chart, then flux-local builds the whole
# kubernetes/ tree the way Flux would. No cluster involved.
- name: Validate charts + Flux tree
run: mise run k8s:validate
integration:
name: Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
persist-credentials: false
- name: Setup Mise
uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0
env:
# mise downloads tools from GitHub releases; anonymous requests hit
# API rate limits with four parallel jobs.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: mise prepare
- name: Start services
# The stack (Ceph image, dev images, k3s) is heavy; the stock runner has
# ~14GB free, so drop the biggest unused toolchains up front.
- name: Free runner disk space
run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true
# Tests boot the apps on the runner; only the infra (CNPG postgres, Ceph
# RGW, mock-oidc, victoria) comes from the cluster — so skip the four
# heavy app images entirely.
- name: Stand up k3d infra
run: |
mise docker:start
- name: Run integration tests
run: mise test:integration
mise k3d:up
mise tilt:ci-infra
- name: Run integration tests against the cluster
run: mise test:integration:k3d
e2e:
name: End-to-end Tests
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
@@ -53,28 +99,26 @@ jobs:
- name: Setup Mise
uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0
env:
# mise downloads tools from GitHub releases; anonymous requests hit
# API rate limits with four parallel jobs.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: mise prepare
- name: Start services & run end-to-end tests
- name: Free runner disk space
run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true
- name: Install Playwright browser
run: pnpm --filter web exec playwright install --with-deps chromium
# Full stack on k3d, then the whole e2e surface — the jest suites
# (it-works, restic-api, yucca-api, orchestration-api) AND the web
# Playwright test — via the same runner developers use locally.
# orchestration-api runs as a separate host process by design.
- name: Stand up the full k3d stack
run: |
mise docker:start
mise dev &
mise test:e2e
mise k3d:up
mise tilt:ci
e2e-web:
name: End-to-end (Web) Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
with:
persist-credentials: false
- name: Setup Mise
uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0
- run: mise prepare
- name: Start services & run end-to-end tests
run: |
mise docker:start
mise dev &
mise test:e2e:web
- name: Run end-to-end tests against the cluster
run: mise test:e2e:k3d
+10
View File
@@ -13,6 +13,16 @@ mise.local.toml
dist/
packages/michael/michael
# k3d/Tilt dev stack — Helm builds subchart snapshots on the fly; the .dev/
# directory holds persistent service data carried over from the compose flow.
.dev/
charts/**/charts/
charts/**/tmpcharts-*/
charts/**/Chart.lock
# air (Go live-reload) temp build output in michael package
packages/michael/tmp/
# OpenTofu / Terraform
# .terraform.lock.hcl IS committed for reproducibility
**/.terraform/
+17 -1
View File
@@ -7,6 +7,20 @@ restic = "0.18.0"
gh = "2.25.0"
"github:git-town/git-town" = "22.4.0"
k3d = "5.8.3"
kubectl = "1.32.2"
helm = "3.17.0"
tilt = "0.34.5"
# Static validation of the k8s surface (mise run k8s:validate, used by CI).
# flux-local runs via `uvx` (see .mise/tasks/k8s/validate): uv auto-fetches a
# Python matching its requires-python, so the host's Python version (3.12 on
# GitHub runners, 3.14 on dev machines) doesn't matter.
kubeconform = "0.8.0"
kustomize = "5.8.1" # flux-local shells out to it
flux2 = "2.7.5" # ...and to the flux CLI
uv = "0.9.18"
# Infrastructure tooling (added for tf/ and ansible/ subtrees)
opentofu = "1.11.5"
terragrunt = "0.99.4"
@@ -30,7 +44,9 @@ depends = ["*:test"]
[tasks."test:integration"]
description = "Run all integration tests"
run = "mise run '*:test:integration' --jobs 1"
# NB: mise flags must precede the task pattern — anything after it is forwarded
# to the tasks themselves (jest/go test choke on a stray --jobs).
run = "mise run --jobs 1 '*:test:integration'"
[tasks."test:e2e:web"]
description = "Run all web e2e tests"
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
#MISE description="Delete k3d cluster"
set -euo pipefail
CLUSTER_NAME="yucca"
if k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then
k3d cluster delete "${CLUSTER_NAME}"
else
echo "Cluster ${CLUSTER_NAME} does not exist"
fi
if k3d registry get k3d-registry.localhost >/dev/null 2>&1; then
k3d registry delete k3d-registry.localhost
fi
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
#MISE description="Recreate k3d cluster"
#MISE depends=["k3d:down", "k3d:up"]
set -euo pipefail
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
#MISE description="Create k3d cluster for local dev"
set -euo pipefail
CLUSTER_NAME="yucca"
if k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then
echo "Cluster ${CLUSTER_NAME} already exists"
else
k3d cluster create --config k3d.yaml
fi
kubectl wait --for=condition=Ready nodes --all --timeout=120s
echo ""
echo "Cluster ready. Context: k3d-${CLUSTER_NAME}"
echo "Registry: k3d-registry.localhost:5000"
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
#MISE description="Statically validate the k8s surface (charts + Flux tree)"
#MISE dir="{{config_root}}"
# Renders every chart (helm template + kubeconform) and builds the whole Flux
# tree the way Flux would (flux-local --enable-helm). No cluster needed; this
# is the CI gate for kubernetes/ and charts/ changes.
#
# NB: don't run this while Tilt is converging — Tilt's helm-deps resource
# rebuilds charts/*/charts/ (rm -rf + dependency build) and races the renders.
set -euo pipefail
LIB_CONSUMERS=(yucca-api yucca-admin-api web michael mock-oidc)
ALL_CHARTS=(yucca-api yucca-admin-api web michael mock-oidc cnpg-cluster ceph-objectuser rook-ceph-cluster)
echo "==> helm dependency build (yucca-common consumers)"
for c in "${LIB_CONSUMERS[@]}"; do
(cd "charts/$c" && helm dependency build >/dev/null)
done
echo "==> helm template + kubeconform"
for c in "${ALL_CHARTS[@]}"; do
ns=yucca
[ "$c" = "rook-ceph-cluster" ] && ns=rook-ceph
# NB: release name must not be YAML-boolean-ish ("y"/"on"/...): it lands in
# labels and kubeconform reads it back as a bool.
helm template yucca "charts/$c" -n "$ns" \
| kubeconform -strict -ignore-missing-schemas - \
&& echo " OK $c"
done
echo "==> kustomize build (Flux entrypoints)"
kustomize build kubernetes/apps >/dev/null && echo " OK kubernetes/apps"
kustomize build kubernetes/flux/repos >/dev/null && echo " OK kubernetes/flux/repos"
kustomize build kubernetes/flux/cluster >/dev/null && echo " OK kubernetes/flux/cluster"
echo "==> flux-local build (full tree, helm rendering as Flux would)"
# Via uvx so uv supplies a Python matching flux-local's requires-python
# (>=3.13) regardless of the host. One retry: flux-local fans out `flux build
# ks` subprocesses which very rarely segfault under load.
flux_local() { uvx --from "flux-local==8.2.0" flux-local "$@"; }
flux_local build all kubernetes --enable-helm --no-enable-dns >/dev/null \
|| flux_local build all kubernetes --enable-helm --no-enable-dns >/dev/null
echo " OK flux-local"
echo "k8s surface: ALL VALID"
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
#MISE description="Run all e2e tests against the local k3d/Tilt stack (orchestration-api runs separately)"
#MISE dir="{{config_root}}"
set -euo pipefail
exec ./packages/e2e/k3d/run.sh
+9 -5
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
#MISE description="Wait for development environment to be ready"
# Targets the compose/`mise dev` flow (the k3d flow has its own readiness
# handling in packages/e2e/k3d/run.sh and does not use this task).
set -e
source "$(dirname "$0")/../../restic-api/env"
source "$(dirname "$0")/../../yucca-api/env"
@@ -9,13 +11,15 @@ echo Ensure dev environment is running before invoking tests.
wait_for_port() {
local port="$1"
local name="$2"
local deadline=$(( $(date +%s) + 30 ))
local timeout="${3:-60}"
local elapsed=0
while ! nc -z localhost "$port" 2>/dev/null; do
if (( $(date +%s) >= deadline )); then
echo "Timed out waiting for $name (:$port)" >&2
exit 1
if [ "$elapsed" -ge "$timeout" ]; then
echo "timed out waiting for $name (localhost:$port) after ${timeout}s" >&2
return 1
fi
sleep 0.5
elapsed=$((elapsed + 1))
done
}
@@ -23,4 +27,4 @@ wait_for_port 8092 mock-oidc-provider
wait_for_port "$RESTIC_API_PORT" restic-api
wait_for_port "$YUCCA_API_PORT" yucca-api
wait_for_port 22676 orchestration-api
wait_for_port 36033 web
wait_for_port "${WEB_PORT:-36033}" web
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
#MISE description="Run all integration tests against the local k3d stack's infra"
#MISE dir="{{config_root}}"
# Kube replacement for the compose-backed `mise docker:start && mise test:integration`:
# the tests boot the apps on this host but take their infrastructure — postgres
# (CNPG), S3 (Ceph RGW), the OIDC provider and the victoria pair — from the k3d
# cluster, port-forwarded to the same localhost ports the compose flow used.
#
# Prereq: mise k3d:up && mise tilt:ci-infra (apps not required)
set -euo pipefail
[ "$(kubectl config current-context)" = "k3d-yucca" ] || {
echo "Not on k3d-yucca. Run: mise k3d:up && mise tilt:ci-infra" >&2; exit 1; }
PF_PIDS=()
cleanup() {
for p in "${PF_PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done
}
trap cleanup EXIT
wait_for() {
local desc="$1"; shift
for _ in $(seq 1 60); do "$@" >/dev/null 2>&1 && return 0; sleep 5; done
echo "timed out waiting for $desc" >&2; return 1
}
echo "==> wait for infra (tilt ci-infra returns before Rook mints the S3 user)"
wait_for "CNPG cluster Ready" kubectl -n yucca wait --for=condition=Ready cluster/yucca-db --timeout=5s
wait_for "Ceph S3 user secret" kubectl -n yucca get secret rook-ceph-object-user-yucca-michael
wait_for "mock-oidc Available" kubectl -n yucca wait --for=condition=Available deploy/yucca-mock-oidc --timeout=5s
echo "==> port-forward infra to the localhost ports the tests expect"
kubectl port-forward -n yucca svc/yucca-db-rw 15432:5432 >/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!)
kubectl port-forward -n rook-ceph svc/rook-ceph-rgw-yucca 9000:80 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!)
kubectl port-forward -n yucca svc/yucca-mock-oidc 8092:8092 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!)
kubectl port-forward -n yucca svc/victoria-metrics 8428:8428 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!)
kubectl port-forward -n yucca svc/victoria-logs 9428:9428 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!)
probe() { (exec 3<>"/dev/tcp/localhost/$1") 2>/dev/null; }
for port in 15432 9000 8092; do
wait_for "localhost:$port" probe "$port"
done
echo "==> export cluster credentials (the per-package env files only set defaults)"
POSTGRES_HOST=localhost
POSTGRES_PORT=15432
POSTGRES_USERNAME="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.username}' | base64 -d)"
POSTGRES_PASSWORD="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.password}' | base64 -d)"
POSTGRES_DATABASE="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.dbname}' | base64 -d)"
export POSTGRES_HOST POSTGRES_PORT POSTGRES_USERNAME POSTGRES_PASSWORD POSTGRES_DATABASE
S3_ENDPOINT=http://localhost:9000
S3_ACCESS_KEY_ID="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-michael -o jsonpath='{.data.AccessKey}' | base64 -d)"
S3_SECRET_ACCESS_KEY="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-michael -o jsonpath='{.data.SecretKey}' | base64 -d)"
S3_REGION=us-east-1
S3_FORCE_PATH_STYLE=true
export S3_ENDPOINT S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY S3_REGION S3_FORCE_PATH_STYLE
# The deployed mock-oidc advertises its in-cluster name as the issuer; the dns
# preload resolves it to the port-forward for every node process (jest + the
# apps it boots). Same split-horizon glue as the e2e runner.
export OIDC_ISSUER=http://yucca-mock-oidc:8092
export OIDC_DEVICE_ISSUER=http://yucca-mock-oidc:8092
export NODE_OPTIONS="--require $PWD/packages/e2e/k3d/hostmap.cjs${NODE_OPTIONS:+ ${NODE_OPTIONS}}"
echo "==> run the integration suites"
mise run test:integration
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
#MISE description="Build + deploy + wait for the k3d stack (tilt ci); optional resource subset"
#MISE dir="{{config_root}}"
set -euo pipefail
if ! kubectl config current-context | grep -q '^k3d-yucca$'; then
echo "Current kube context is not k3d-yucca. Run: mise k3d:up" >&2
exit 1
fi
exec tilt ci --timeout 1800s "$@"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
#MISE description="tilt ci for the infra subset only (no app images) — what integration tests need"
#MISE dir="{{config_root}}"
# The closure of every non-app resource: chart repos, operators, the Ceph dev
# cluster + RGW user, CNPG database, mock-oidc and the victoria pair. Skipping
# the four heavy app images (yucca-api/admin/web/michael) saves ~10 min in CI;
# integration tests boot those apps on the host themselves.
set -euo pipefail
if ! kubectl config current-context | grep -q '^k3d-yucca$'; then
echo "Current kube context is not k3d-yucca. Run: mise k3d:up" >&2
exit 1
fi
exec tilt ci --timeout 1800s \
cloudnative-pg-repo rook-release-repo victoriametrics-repo helm-deps \
cloudnative-pg rook-ceph-operator rook-ceph-cluster \
yucca-object-user yucca-database yucca-mock-oidc \
yucca-victoria-metrics yucca-victoria-logs
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
#MISE description="Tear down Tilt resources"
set -euo pipefail
tilt down --delete-namespaces "$@"
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
#MISE description="Start Tilt against the local k3d cluster"
set -euo pipefail
if ! kubectl config current-context | grep -q '^k3d-yucca$'; then
echo "Current kube context is not k3d-yucca. Run: mise k3d:up"
exit 1
fi
tilt up "$@"
+5
View File
@@ -31,5 +31,10 @@ yarn.lock
ansible/
tf/
# Helm charts: templates are Go-templated YAML (not parseable by prettier),
# and values/Chart files follow helm conventions. Validated by helm template
# + kubeconform via `mise k8s:validate` instead.
charts/
# auto-generated
yaak/
+30 -6
View File
@@ -18,7 +18,7 @@ If necessary, copy `.env.example` to `.env` and customise.
Then use mise:
```bash
mise dev # install deps, prep environment, start servers
mise dev # install deps, prep environment, start servers (compose-based)
mise check # lint, format check, svelte check
@@ -28,13 +28,37 @@ mise test:e2e # e2e tests
mise test:e2e:web # e2e web tests
```
### Running on k3d + Tilt (Kubernetes)
An alternative k8s-based dev flow mirrors the eventual prod topology (Helm charts, CloudNativePG, Rook-Ceph object storage, in-cluster service discovery). All required tools (k3d, kubectl, helm, tilt) are installed via mise.
```bash
mise k3d:up # create local k3d cluster + registry
mise tilt:up # start Tilt; builds images, renders charts, port-forwards
# edit code — live_update syncs into running pods
mise tilt:down # stop Tilt
mise k3d:down # delete cluster
```
Ports forwarded to localhost:
- `5173` web, `3020` yucca-api, `3030` yucca-admin-api, `3010` michael
- `8092` mock-oidc, `9000` ceph rgw (S3)
- `8428` victoria-metrics, `9428` victoria-logs
Tilt deploys the **same per-app charts the Flux tree uses** — it reads the
HelmReleases under [`kubernetes/apps`](./kubernetes) to discover what to deploy,
then builds/live-updates the images. Charts live in `charts/` (a `yucca-common`
library + per-service charts). See [`kubernetes/README.md`](./kubernetes/README.md).
## Infrastructure
| Start here | Path | Purpose |
| ---------------------------------------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
| [`ansible/ceph/README.md`](./ansible/ceph/README.md) | `ansible/ceph/` | Ansible automation for Ceph clusters (sietch, painbox). Deploys + operates via cephadm on bare-metal and Hetzner. |
| [`tf/README.md`](./tf/README.md) | `tf/` | Terraform/OpenTofu authority for cluster identity, 1P secret items, rendered Ansible inventories. Terragrunt multi-env (`deployment/<env>/<stack>/`). |
| (coming in follow-up) | `kubernetes/` | Flux GitOps surface for the (future) Talos K8s cluster. |
| Start here | Path | Purpose |
| ---------------------------------------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| [`ansible/ceph/README.md`](./ansible/ceph/README.md) | `ansible/ceph/` | Ansible automation for Ceph clusters (sietch, painbox). Deploys + operates via cephadm on bare-metal and Hetzner. |
| [`tf/README.md`](./tf/README.md) | `tf/` | Terraform/OpenTofu authority for cluster identity, 1P secret items, rendered Ansible inventories. Terragrunt multi-env (`deployment/<env>/<stack>/`). |
| [`kubernetes/README.md`](./kubernetes/README.md) | `kubernetes/` | Flux GitOps surface (apps/components/flux/bootstrap) for the (future) Talos K8s cluster. Per-app HelmReleases over the in-repo `charts/`; mirrored locally by Tilt. |
**Secrets are managed via the `yucca_tf_*` 1Password vaults.** Runtime reads use a
read-only service account; TF writes use a superuser service account. See
+377
View File
@@ -0,0 +1,377 @@
# Yucca local dev on k3d + Tilt + Helm.
#
# Source of truth = the Flux tree under kubernetes/. Tilt derives EVERYTHING it
# deploys from there (see discover_apps below):
# - GitRepository-sourced HelmReleases -> the in-repo charts/<svc>, rendered
# with their dev defaults (charts/*/values.yaml) and live-updated with the
# locally-built images.
# - HelmRepository-sourced HelmReleases (cnpg, rook, victoria-*) -> installed
# at the exact chart version + values pinned in the HelmRelease, from the
# HelmRepositories declared in kubernetes/flux/repos/.
# APP_WIRING below carries only the dev-specific concerns Flux doesn't have:
# which locally-built image to inject, deploy ordering, and pod-readiness quirks.
#
# Service names are pinned via fullnameOverride in each chart, so in-cluster DNS
# is identical whether a chart is rendered here (release == resource name) or by
# Flux (per-app release names).
load('ext://helm_resource', 'helm_resource', 'helm_repo')
load('ext://namespace', 'namespace_create')
# Gate: only talk to the local k3d cluster. Prevents accidental prod deploys.
allow_k8s_contexts('k3d-yucca')
namespace_create('yucca')
# ---------------------------------------------------------------------------
# Optional dev secrets: a gitignored .env at the repo root (KEY=VALUE; values
# may be 1Password `op://` references, resolved here via `op read`). When
# present it becomes the yucca-dev-env Secret, layered onto yucca-api as the
# last envFrom source (last source wins for duplicate keys). read_file watches
# the path, so creating/editing .env retriggers automatically. Absent .env
# (CI, fresh clones) leaves the committed mock-oidc dev fixtures in charge.
# ---------------------------------------------------------------------------
# Env vars the chart pins as explicit container env — explicit env always
# beats envFrom, so these .env keys must override through Helm values instead.
DEV_ENV_VALUE_KEYS = {
'OIDC_ISSUER': 'oidcIssuer',
'OIDC_REDIRECT_URI': 'oidcRedirectUri',
'OIDC_LOGOUT_REDIRECT_URI': 'oidcLogoutRedirectUri',
}
def load_dev_env():
env = {}
for line in str(read_file('.env', default='')).splitlines():
line = line.strip()
if line.startswith('export '):
line = line[len('export '):].lstrip()
if not line or line.startswith('#') or '=' not in line:
continue
key, _, value = line.partition('=')
env[key.strip()] = value.strip().strip('"').strip("'")
# OP_ACCOUNT picks the 1Password account on multi-account machines. It's
# loader config, not app env, so it never reaches the cluster.
account = env.pop('OP_ACCOUNT', '')
for key in env.keys():
if env[key].startswith('op://'):
cmd = ['op', 'read', '--no-newline'] + (['--account', account] if account else []) + [env[key]]
# quiet/echo_off: keep resolved secrets out of the Tilt log. Fails
# loudly (aborting the Tiltfile) if op is missing or signed out.
env[key] = str(local(cmd, quiet=True, echo_off=True))
return env
DEV_ENV = load_dev_env()
if DEV_ENV:
k8s_yaml(encode_yaml({
'apiVersion': 'v1',
'kind': 'Secret',
'metadata': {'name': 'yucca-dev-env', 'namespace': 'yucca'},
'stringData': DEV_ENV,
}))
k8s_resource(objects=['yucca-dev-env:secret'], new_name='dev-env', labels=['helm'])
# ---------------------------------------------------------------------------
# Images. Built locally and injected into each app's Helm release via image_keys
# (image.repository/image.tag). Edits are live-synced into the running pods.
# ---------------------------------------------------------------------------
docker_build(
'yucca-api',
context='.',
dockerfile='packages/yucca-api/Dockerfile',
target='dev',
# Rebuild only on package.json/lockfile/Dockerfile changes; src edits are
# handled by the syncs below (nest --watch picks them up in-pod).
only=[
'./pnpm-workspace.yaml',
'./pnpm-lock.yaml',
'./package.json',
'./.npmrc',
'./packages',
],
ignore=[
'**/node_modules',
'**/dist',
'**/.svelte-kit',
'packages/michael',
'packages/e2e',
],
live_update=[
sync('./packages/yucca-api', '/app/packages/yucca-api'),
sync('./packages/common', '/app/packages/common'),
run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']),
],
)
docker_build(
'web',
context='.',
dockerfile='packages/web/Dockerfile',
target='dev',
only=[
'./pnpm-workspace.yaml',
'./pnpm-lock.yaml',
'./package.json',
'./.npmrc',
'./packages',
],
ignore=[
'**/node_modules',
'**/dist',
'**/.svelte-kit',
'packages/michael',
'packages/e2e',
],
live_update=[
sync('./packages/web', '/app/packages/web'),
sync('./packages/common', '/app/packages/common'),
sync('./packages/yucca-api-client', '/app/packages/yucca-api-client'),
sync('./packages/yucca-sdk', '/app/packages/yucca-sdk'),
run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']),
run('cd /app && pnpm --filter @futo-org/backups-api-client build', trigger=['./packages/yucca-api-client/src']),
run('cd /app && pnpm --filter @futo-org/backups-orchestrator-ui build', trigger=['./packages/yucca-sdk/orchestration-ui/src']),
run('cd /app && pnpm --filter web lingui:compile', trigger=['./packages/web/src/locales']),
],
)
docker_build(
'michael',
context='.',
dockerfile='packages/michael/Dockerfile',
target='dev',
only=['./packages/michael'],
live_update=[
sync('./packages/michael', '/src'),
run('cd /src && go mod download', trigger=['./packages/michael/go.sum']),
],
)
docker_build(
'yucca-admin-api',
context='.',
dockerfile='packages/yucca-admin-api/Dockerfile',
target='dev',
only=[
'./pnpm-workspace.yaml',
'./pnpm-lock.yaml',
'./package.json',
'./.npmrc',
'./packages',
],
ignore=[
'**/node_modules',
'**/dist',
'**/.svelte-kit',
'packages/michael',
'packages/e2e',
],
live_update=[
sync('./packages/yucca-admin-api', '/app/packages/yucca-admin-api'),
sync('./packages/common', '/app/packages/common'),
# yucca-admin-api/src/schema is a symlink into yucca-api; keep it synced.
sync('./packages/yucca-api', '/app/packages/yucca-api'),
run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']),
],
)
# mock-oidc-provider has no dev target (config-only via env); a plain build is
# enough — it rarely changes and is reconfigured through Helm values.
docker_build(
'mock-oidc-provider',
context='.',
dockerfile='packages/mock-oidc-provider/Dockerfile',
only=[
'./pnpm-workspace.yaml',
'./pnpm-lock.yaml',
'./package.json',
'./.npmrc',
'./packages/mock-oidc-provider',
],
ignore=[
'**/node_modules',
'**/dist',
],
)
# ---------------------------------------------------------------------------
# First-party Helm charts that depend on the yucca-common library need their
# subchart snapshot built before `helm upgrade` can render them.
# ---------------------------------------------------------------------------
local_resource(
'helm-deps',
cmd='rm -rf charts/yucca-api/charts charts/yucca-admin-api/charts charts/web/charts charts/michael/charts charts/mock-oidc/charts && for d in charts/yucca-api charts/yucca-admin-api charts/web charts/michael charts/mock-oidc; do (cd $d && helm dependency build); done',
deps=[
'charts/yucca-api',
'charts/yucca-admin-api',
'charts/web',
'charts/michael',
'charts/mock-oidc',
'charts/yucca-common',
],
# `helm dependency build` rewrites these; if Tilt watches them we re-enter
# an infinite rebuild loop.
ignore=[
'charts/**/charts',
'charts/**/charts/**',
'charts/**/tmpcharts-*',
'charts/**/tmpcharts-*/**',
'charts/**/Chart.lock',
],
labels=['helm'],
)
# ---------------------------------------------------------------------------
# Deploy everything the Flux tree declares. The HelmRelease tree is the source
# of truth for WHAT runs (apps, operators, chart versions, remote values); the
# map below carries only the DEV concerns Flux doesn't know about.
# ---------------------------------------------------------------------------
APP_WIRING = {
# name (== HelmRelease metadata.name) build image ref resource_deps
# dev_env: receives the .env override Secret (see load_dev_env above).
'yucca-api': {'build': 'yucca-api', 'deps': ['yucca-database', 'yucca-mock-oidc', 'yucca-michael'], 'dev_env': True},
'yucca-admin-api': {'build': 'yucca-admin-api', 'deps': ['yucca-database', 'yucca-mock-oidc']},
'yucca-web': {'build': 'web', 'deps': ['yucca-api']},
'yucca-michael': {'build': 'michael', 'deps': ['yucca-object-user']},
'yucca-mock-oidc': {'build': 'mock-oidc-provider', 'deps': []},
'yucca-database': {'build': None, 'deps': ['cloudnative-pg']},
# The CephObjectStoreUser creates no pods (just a Secret once Rook mints the
# RGW user), so Tilt's pod tracking would hang at "pending". Mark ready on
# apply; michael still waits on this resource for ordering.
'yucca-object-user': {'build': None, 'deps': ['rook-ceph-cluster'], 'pod_readiness': 'ignore'},
# Rook spins up transient mon/osd "canary" pods and deletes them; Tilt's
# pod tracking misreads those deletions as failures. Ignore pod readiness
# here — real convergence is still gated downstream (object-user -> michael
# only go ready once the RGW + user secret actually exist).
'rook-ceph-cluster': {'build': None, 'deps': ['rook-ceph-operator'], 'pod_readiness': 'ignore'},
# Remote-chart operators/infra (HelmRepository-sourced).
'cloudnative-pg': {'build': None, 'deps': []},
'rook-ceph-operator': {'build': None, 'deps': []},
'yucca-victoria-metrics': {'build': None, 'deps': []},
'yucca-victoria-logs': {'build': None, 'deps': []},
}
def discover_helm_repos():
"""HelmRepository name -> URL, from kubernetes/flux/repos/."""
repos = {}
for path in listdir('kubernetes/flux/repos'):
if not path.endswith('.yaml'):
continue
doc = read_yaml(path)
if doc and doc.get('kind') == 'HelmRepository':
repos[doc['metadata']['name']] = doc['spec']['url']
return repos
def discover_apps():
"""All HelmReleases under kubernetes/apps, split by chart source kind."""
local_apps, remote_apps = [], []
for path in listdir('kubernetes/apps', recursive=True):
if not path.endswith('/helmrelease.yaml'):
continue
hr = read_yaml(path)
if not hr or hr.get('kind') != 'HelmRelease':
continue
chart_spec = hr['spec']['chart']['spec']
source = chart_spec.get('sourceRef', {})
chart = chart_spec.get('chart', '')
name = hr['metadata']['name']
namespace = hr['metadata'].get('namespace', 'yucca')
if source.get('kind') == 'GitRepository' and chart.startswith('charts/'):
# In-repo chart: dev renders it with its values.yaml defaults; the
# HelmRelease's .spec.values are the prod-side overrides.
local_apps.append(struct(name=name, namespace=namespace, chart=chart))
elif source.get('kind') == 'HelmRepository':
# Remote chart: dev installs the exact version + values Flux would.
remote_apps.append(struct(
name=name,
namespace=namespace,
chart=chart,
version=chart_spec.get('version', ''),
repo=source['name'],
values=hr['spec'].get('values', {}),
))
return local_apps, remote_apps
def wiring_for(app):
wiring = APP_WIRING.get(app.name)
if wiring == None:
fail("HelmRelease '%s' (%s) has no Tilt dev wiring — add it to APP_WIRING in the Tiltfile" % (app.name, app.chart))
return wiring
LOCAL_APPS, REMOTE_APPS = discover_apps()
HELM_REPOS = discover_helm_repos()
for repo_name, url in HELM_REPOS.items():
helm_repo('%s-repo' % repo_name, url, labels=['helm'])
for app in REMOTE_APPS:
wiring = wiring_for(app)
flags = ['--create-namespace']
if app.version:
flags += ['--version', app.version]
# Pass the HelmRelease's values verbatim (one --set-json per top-level key).
for key in sorted(app.values.keys()):
flags += ['--set-json', '%s=%s' % (key, str(encode_json(app.values[key])).rstrip('\n'))]
helm_resource(
app.name,
'%s-repo/%s' % (app.repo, app.chart),
namespace=app.namespace,
flags=flags,
resource_deps=['%s-repo' % app.repo] + wiring['deps'],
labels=['helm'],
pod_readiness=wiring.get('pod_readiness', ''),
)
for app in LOCAL_APPS:
wiring = wiring_for(app)
builds = [wiring['build']] if wiring['build'] else []
flags = ['--timeout=10m']
extra_deps = []
if DEV_ENV and wiring.get('dev_env'):
flags += ['--set-json', 'extraEnvFrom=[{"secretRef":{"name":"yucca-dev-env"}}]']
for key, value_path in DEV_ENV_VALUE_KEYS.items():
if key in DEV_ENV:
flags += ['--set-string', '%s=%s' % (value_path, DEV_ENV[key])]
extra_deps = ['dev-env']
helm_resource(
app.name,
app.chart,
namespace=app.namespace,
flags=flags,
image_deps=builds,
image_keys=[('image.repository', 'image.tag')] if builds else [],
resource_deps=['helm-deps'] + wiring['deps'] + extra_deps,
labels=['app'],
deps=[app.chart, 'charts/yucca-common'],
pod_readiness=wiring.get('pod_readiness', ''),
)
# ---------------------------------------------------------------------------
# Port-forwards. helm_resource bundles a release into one Tilt resource, so a
# single local_resource with raw kubectl tunnels gives each service its own.
# ---------------------------------------------------------------------------
local_resource(
'port-forwards',
serve_cmd='''trap 'kill 0' EXIT
kubectl port-forward -n yucca svc/yucca-api 3020:3020 &
kubectl port-forward -n yucca svc/yucca-admin-api 3030:3030 &
kubectl port-forward -n yucca svc/yucca-web 5173:5173 &
kubectl port-forward -n yucca svc/yucca-michael 3010:3010 &
kubectl port-forward -n yucca svc/yucca-mock-oidc 8092:8092 &
kubectl port-forward -n rook-ceph svc/rook-ceph-rgw-yucca 9000:80 &
kubectl port-forward -n yucca svc/victoria-metrics 8428:8428 &
kubectl port-forward -n yucca svc/victoria-logs 9428:9428 &
wait''',
resource_deps=['yucca-api', 'yucca-web', 'yucca-michael', 'yucca-mock-oidc'],
labels=['app'],
links=[
link('http://localhost:5173', 'web'),
link('http://localhost:3020', 'yucca-api'),
link('http://localhost:3030', 'yucca-admin-api'),
link('http://localhost:3010', 'michael'),
link('http://localhost:8092', 'mock-oidc'),
link('http://localhost:9000', 'ceph rgw (s3)'),
link('http://localhost:8428', 'victoria-metrics'),
link('http://localhost:9428', 'victoria-logs'),
],
)
+9
View File
@@ -0,0 +1,9 @@
apiVersion: v2
name: ceph-objectuser
description: >-
CephObjectStoreUser for the dev Rook-Ceph object store. Creates a full RGW S3
user (can create/manage buckets) and writes its credentials Secret into this
namespace. michael uses it (one bucket per restic repository). DEV ONLY.
type: application
version: 0.1.0
appVersion: "0.0.1"
@@ -0,0 +1,8 @@
apiVersion: ceph.rook.io/v1
kind: CephObjectStoreUser
metadata:
name: {{ .Values.userName }}
spec:
store: {{ .Values.store }}
clusterNamespace: {{ .Values.clusterNamespace }}
displayName: {{ .Values.userName }}
+7
View File
@@ -0,0 +1,7 @@
# RGW user for michael. Rook writes a Secret named
# "rook-ceph-object-user-<store>-<userName>" into THIS namespace with keys
# AccessKey / SecretKey. michael (charts/michael) consumes them.
userName: michael
# CephObjectStore name + the namespace where the Rook cluster/objectstore lives.
store: yucca
clusterNamespace: rook-ceph
+6
View File
@@ -0,0 +1,6 @@
apiVersion: v2
name: cnpg-cluster
description: CloudNativePG Cluster for Yucca (consumes the cnpg operator's CRDs)
type: application
version: 0.1.0
appVersion: "0.0.1"
@@ -0,0 +1,12 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: {{ .Values.clusterName }}
spec:
instances: {{ .Values.instances }}
storage:
size: {{ .Values.storage }}
bootstrap:
initdb:
database: {{ .Values.database }}
owner: {{ .Values.owner }}
+8
View File
@@ -0,0 +1,8 @@
# CNPG Cluster. The name is the stable in-cluster identifier; yucca-api and
# yucca-admin-api derive their POSTGRES_* env from the "<clusterName>-app" secret
# that CNPG generates, so keep this consistent across dev and prod.
clusterName: yucca-db
instances: 1
storage: 1Gi
database: yucca
owner: yucca
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: michael
description: Yucca backup/restore service (Go)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.deployment" . }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.secret" . }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.service" . }}
+76
View File
@@ -0,0 +1,76 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name (dev == prod).
fullnameOverride: yucca-michael
image:
repository: k3d-registry.localhost:5000/michael
tag: dev
pullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3010
service:
type: ClusterIP
# DEV FIXTURE — the public half of the project's well-known local-dev keypair
# (see charts/yucca-api/values.yaml + .mise/tasks/*/env). Prod replaces this
# with an ExternalSecret.
secretData:
# michael (Go) verifies ES256 JWTs from yucca-api with this public key.
JWT_PUBLIC_KEY: |
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpLmwUSBO0p7P1UvGReVxFTvAsCfg
GS7NQtJ3AnJkYaigO1MS5J59I4uRXCmpLtcwTocUGHMRVZrGLQMWdZY4DQ==
-----END PUBLIC KEY-----
env:
- name: RESTIC_API_PORT
value: "3010"
- name: LOG_LEVEL
value: debug
# S3 object store = Rook-Ceph RGW. michael creates one bucket per restic
# repository, so it needs a full RGW user (CephObjectStoreUser via
# charts/ceph-objectuser), NOT a bucket-scoped ObjectBucketClaim. Rook writes
# the user's keys into this namespace as rook-ceph-object-user-<store>-<user>.
- name: S3_ENDPOINT
value: http://rook-ceph-rgw-yucca.rook-ceph.svc:80
- name: S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: rook-ceph-object-user-yucca-michael
key: AccessKey
- name: S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: rook-ceph-object-user-yucca-michael
key: SecretKey
- name: S3_REGION
value: us-east-1
- name: S3_FORCE_PATH_STYLE
value: "true"
- name: OTLP_METRICS_ENDPOINT
value: victoria-metrics:8428
- name: OTLP_METRICS_URL_PATH
value: /opentelemetry/v1/metrics
- name: OTLP_LOGS_ENDPOINT
value: victoria-logs:9428
- name: OTLP_LOGS_URL_PATH
value: /insert/opentelemetry/v1/logs
envFrom:
- secretRef:
name: yucca-michael
# Probes keep `tilt ci`/Flux honest: michael runs under air in dev, which keeps
# the container "Running" even when the binary fatals on boot. The TCP probe
# surfaces that as NotReady. No livenessProbe (air restarts are normal in dev).
startupProbe:
tcpSocket: { port: http }
periodSeconds: 2
failureThreshold: 90
readinessProbe:
tcpSocket: { port: http }
periodSeconds: 10
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: mock-oidc
description: Mock OIDC provider for local dev (homegrown packages/mock-oidc-provider)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
@@ -0,0 +1 @@
{{- include "yucca-common.deployment" . }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.service" . }}
+53
View File
@@ -0,0 +1,53 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name (dev == prod).
# yucca-api/admin-api reference this as their OIDC issuer host.
fullnameOverride: yucca-mock-oidc
image:
repository: k3d-registry.localhost:5000/mock-oidc-provider
tag: dev
pullPolicy: IfNotPresent
# Container listens on 80 (see packages/mock-oidc-provider); exposed in-cluster
# and via port-forward as 8092 to match yucca-api's oidcIssuer.
ports:
- name: http
containerPort: 80
servicePort: 8092
service:
type: ClusterIP
env:
- name: PORT
value: "80"
# iss claim / discovery base — must match yucca-api's OIDC_ISSUER
# (http://yucca-mock-oidc:8092) so issued tokens validate in-cluster.
- name: ISSUER
value: http://yucca-mock-oidc:8092
- name: CLIENT_ID
value: "client ID"
- name: CLIENT_SECRET
value: "client secret"
- name: DEVICE_CLIENT_ID
value: "device client ID"
# Browser-facing callbacks resolve via the Tilt port-forwards.
- name: REDIRECT_URI
value: http://localhost:5173/api/auth/oidc/callback
- name: POST_LOGOUT_REDIRECT_URI
value: http://localhost:5173
- name: ADMIN_REDIRECT_URI
value: http://localhost:3030/api/auth/oidc/callback
- name: ADMIN_POST_LOGOUT_REDIRECT_URI
value: http://localhost:3030
# Probe the discovery document — it's what every consumer (yucca-api, the e2e,
# browsers) needs working.
startupProbe:
httpGet: { path: /.well-known/openid-configuration, port: http }
periodSeconds: 2
failureThreshold: 60
readinessProbe:
httpGet: { path: /.well-known/openid-configuration, port: http }
periodSeconds: 10
+9
View File
@@ -0,0 +1,9 @@
apiVersion: v2
name: rook-ceph-cluster
description: >-
Minimal single-node Rook-Ceph cluster + S3 object store for LOCAL DEV ONLY.
Replaces the dev MinIO object store. Not for production — prod storage is a
completely separate Ceph (see ansible/ceph + tf/).
type: application
version: 0.1.0
appVersion: "1.20.0"
@@ -0,0 +1,46 @@
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
name: {{ .Values.clusterName }}
spec:
dataDirHostPath: {{ .Values.dataDirHostPath }}
cephVersion:
image: {{ .Values.cephImage }}
# single-node, single-replica dev cluster is not a supported topology
allowUnsupported: true
mon:
count: {{ .Values.mon.count }}
allowMultiplePerNode: true
mgr:
count: {{ .Values.mgr.count }}
allowMultiplePerNode: true
dashboard:
enabled: {{ .Values.dashboard.enabled }}
# Trim the footprint for a laptop-sized dev cluster.
crashCollector:
disable: true
cephConfig:
global:
# no redundancy on a single OSD
osd_pool_default_size: "1"
osd_pool_default_min_size: "1"
mon_warn_on_pool_no_redundancy: "false"
mon_allow_pool_size_one: "true"
# Single-OSD dev fix: the PG autoscaler starts pools at pg_num=1, but Rook
# sets pg_num_min=8 on the RGW system pools (e.g. .rgw.root) -> EINVAL
# "pg_num_min 8 > pg_num 1". Disable autoscaling and default new pools to
# 8 PGs so the object store's pools are created at pg_num=8.
osd_pool_default_pg_autoscale_mode: "off"
osd_pool_default_pg_num: "8"
osd_pool_default_pgp_num: "8"
storage:
useAllNodes: true
useAllDevices: false
# Loop devices must be named explicitly (see values.yaml). Requires
# allowLoopDevices=true on the operator.
devices:
- name: {{ .Values.storage.device }}
healthCheck:
daemonHealth:
mon:
interval: 45s
@@ -0,0 +1,25 @@
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
name: {{ .Values.objectStore.name }}
spec:
metadataPool:
failureDomain: osd
replicated:
size: 1
requireSafeReplicaSize: false
dataPool:
failureDomain: osd
replicated:
size: 1
requireSafeReplicaSize: false
preservePoolsOnDelete: false
gateway:
port: {{ .Values.objectStore.gateway.port }}
instances: {{ .Values.objectStore.gateway.instances }}
# Allow CephObjectStoreUser CRs in the app namespace (so the user's S3 secret
# is written there for michael to consume). See charts/ceph-objectuser.
allowUsersInNamespaces:
{{- range .Values.objectStore.allowUsersInNamespaces }}
- {{ . }}
{{- end }}
@@ -0,0 +1,73 @@
{{- if .Values.loopDevice.enabled }}
# DEV ONLY. k3d nodes have no spare block device, and Rook OSDs need raw block
# storage (k3d's local-path provisioner is filesystem-only). This privileged
# DaemonSet creates a sparse image on the node and losetup-attaches it to a
# FIXED loop device (storage.device / loopDevice.device) that the CephCluster
# names explicitly. The container stays alive so the attachment persists.
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: {{ .Values.clusterName }}-loop-device
labels:
app.kubernetes.io/name: rook-ceph-loop-device
spec:
selector:
matchLabels:
app.kubernetes.io/name: rook-ceph-loop-device
template:
metadata:
labels:
app.kubernetes.io/name: rook-ceph-loop-device
spec:
hostPID: true
containers:
- name: loop-device
image: {{ .Values.loopDevice.image }}
securityContext:
privileged: true
command: ["/bin/sh", "-c"]
args:
- |
set -eu
apk add --no-cache util-linux >/dev/null 2>&1 || true
IMG="{{ .Values.loopDevice.path }}"
DEV="{{ .Values.loopDevice.device }}"
mkdir -p "$(dirname "$IMG")"
[ -f "$IMG" ] || truncate -s {{ .Values.loopDevice.sizeGiB }}G "$IMG"
# Attach the image to the FIXED device the CephCluster references.
# Loop attachments live in the HOST kernel (shared by every k3d
# node), so they survive cluster re-creation: after a k3d:reset,
# DEV is typically still bound to the PREVIOUS cluster's now-
# deleted image — which also makes Rook skip it (stale bluestore
# signature). Clean all stale state before attaching:
if losetup "$DEV" >/dev/null 2>&1; then
back="$(losetup --noheadings --output BACK-FILE "$DEV" 2>/dev/null | xargs || true)"
# detach unless it's a live attachment of exactly our image
[ "$back" = "$IMG" ] || losetup -d "$DEV" || true
fi
# our image attached on some other device = corruption hazard
for d in $(losetup -j "$IMG" | cut -d: -f1); do
[ "$d" = "$DEV" ] || losetup -d "$d" || true
done
if ! losetup "$DEV" >/dev/null 2>&1; then
[ -e "$DEV" ] || mknod "$DEV" b 7 "${DEV#/dev/loop}"
losetup "$DEV" "$IMG"
fi
echo "loop device for Rook OSD: $(losetup -j "$IMG" | cut -d: -f1) ($IMG)"
# keep the container (and thus the loop attachment) alive
while true; do sleep 3600; done
volumeMounts:
- name: dev
mountPath: /dev
mountPropagation: Bidirectional
- name: rook-data
mountPath: {{ dir .Values.loopDevice.path }}
volumes:
- name: dev
hostPath:
path: /dev
- name: rook-data
hostPath:
path: {{ dir .Values.loopDevice.path }}
type: DirectoryOrCreate
{{- end }}
+44
View File
@@ -0,0 +1,44 @@
# Minimal single-node Rook-Ceph for local dev (k3d). DEV ONLY.
clusterName: rook-ceph
dataDirHostPath: /var/lib/rook
# Pin to match the rook-ceph operator appVersion (Renovate keeps these in step).
cephImage: quay.io/ceph/ceph:v19.2.2
mon:
count: 1
mgr:
count: 1
dashboard:
enabled: false
storage:
# Rook does NOT auto-select loop devices via useAllDevices (even with
# allowLoopDevices on the operator) — they must be named explicitly. This is
# the loop device attached by the DaemonSet below. A HIGH minor number: the
# kernel hands out the lowest free loop devices, so k3s/containerd will have
# claimed loop0..N on a fresh node — loop100 is reliably ours.
device: /dev/loop100
# S3 object store (CephObjectStore + RGW).
objectStore:
name: yucca
region: us-east-1
gateway:
port: 80
instances: 1
# Namespaces allowed to host CephObjectStoreUser CRs (michael's S3 user lives
# in the yucca namespace so its secret is written there).
allowUsersInNamespaces:
- yucca
# k3d has no spare block device, so synthesize one with a loopback file. This
# privileged DaemonSet truncates a sparse image on the node and losetup-attaches
# it; Rook's OSD then consumes it. Disable if your nodes have real disks.
loopDevice:
enabled: true
image: alpine:3.21
sizeGiB: 20
# where the backing image file lives on the node (under dataDirHostPath)
path: /var/lib/rook/osd-loop.img
# fixed device the image is attached to (must match storage.device above)
device: /dev/loop100
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: web
description: Yucca web UI (SvelteKit SSR)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.deployment" . }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.service" . }}
+38
View File
@@ -0,0 +1,38 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name (dev == prod).
fullnameOverride: yucca-web
image:
repository: k3d-registry.localhost:5000/web
tag: dev
pullPolicy: IfNotPresent
ports:
- name: http
containerPort: 5173
service:
type: ClusterIP
env:
- name: NODE_ENV
value: development
- name: PUBLIC_API_URL
value: http://yucca-api:3020
- name: YUCCA_API_URL
value: http://yucca-api:3020/
# Probes keep `tilt ci`/Flux honest: vite binds the port even when SSR is
# broken (a stale workspace lib once made every page a 500 while the pod looked
# Ready), so probe with a real GET /. Generous timeouts — the first request
# triggers vite's initial compile.
startupProbe:
httpGet: { path: /, port: http }
periodSeconds: 5
timeoutSeconds: 10
failureThreshold: 60
readinessProbe:
httpGet: { path: /, port: http }
periodSeconds: 15
timeoutSeconds: 10
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: yucca-admin-api
description: Yucca admin API (NestJS)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
@@ -0,0 +1,12 @@
{{- $_ := set .Values "env" (concat .Values.env (list
(dict "name" "OIDC_ADMIN_ISSUER" "value" .Values.oidcIssuer)
(dict "name" "OIDC_ADMIN_REDIRECT_URI" "value" .Values.oidcRedirectUri)
(dict "name" "OIDC_ADMIN_LOGOUT_REDIRECT_URI" "value" .Values.oidcLogoutRedirectUri)
(dict "name" "POSTGRES_HOST" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "host")))
(dict "name" "POSTGRES_PORT" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "port")))
(dict "name" "POSTGRES_DATABASE" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "dbname")))
(dict "name" "POSTGRES_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "username")))
(dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password")))
)) }}
{{- $_ := set .Values "envFrom" (list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .)))) }}
{{- include "yucca-common.deployment" . }}
@@ -0,0 +1 @@
{{- include "yucca-common.secret" . }}
@@ -0,0 +1 @@
{{- include "yucca-common.service" . }}
+58
View File
@@ -0,0 +1,58 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name (dev == prod).
fullnameOverride: yucca-admin-api
image:
repository: k3d-registry.localhost:5000/yucca-admin-api
tag: dev
pullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3030
service:
type: ClusterIP
# CNPG-managed postgres Cluster name (shares yucca-api's database)
postgresClusterName: yucca-db
# OIDC provider in-cluster service (must match the issuer mock-oidc advertises)
oidcIssuer: http://yucca-mock-oidc:8092
oidcRedirectUri: http://localhost:3030/api/auth/oidc/callback
oidcLogoutRedirectUri: http://localhost:3030
# Static dev secrets (overridden in prod via ExternalSecret)
secretData:
OIDC_ADMIN_CLIENT_ID: "client ID"
OIDC_ADMIN_CLIENT_SECRET: "client secret"
env:
- name: NODE_ENV
value: development
- name: YUCCA_ADMIN_API_PORT
value: "3030"
- name: LOG_LEVEL
value: debug
- name: OIDC_ADMIN_ALLOW_INSECURE
value: "true"
- name: OTLP_METRICS_ENDPOINT
value: victoria-metrics:8428
- name: OTLP_METRICS_URL_PATH
value: /opentelemetry/v1/metrics
- name: OTLP_LOGS_ENDPOINT
value: victoria-logs:9428
- name: OTLP_LOGS_URL_PATH
value: /insert/opentelemetry/v1/logs
# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process
# still counts as Ready (this masked two real bugs). The startupProbe budgets
# for the dev watcher's first boot; no livenessProbe so dev never restart-loops.
startupProbe:
tcpSocket: { port: http }
periodSeconds: 5
failureThreshold: 60
readinessProbe:
tcpSocket: { port: http }
periodSeconds: 10
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: yucca-api
description: Yucca REST API (NestJS)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
@@ -0,0 +1,16 @@
{{- $_ := set .Values "env" (concat .Values.env (list
(dict "name" "OIDC_ISSUER" "value" .Values.oidcIssuer)
(dict "name" "OIDC_REDIRECT_URI" "value" .Values.oidcRedirectUri)
(dict "name" "OIDC_LOGOUT_REDIRECT_URI" "value" .Values.oidcLogoutRedirectUri)
(dict "name" "POSTGRES_HOST" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "host")))
(dict "name" "POSTGRES_PORT" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "port")))
(dict "name" "POSTGRES_DATABASE" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "dbname")))
(dict "name" "POSTGRES_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "username")))
(dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password")))
)) }}
{{- /* extraEnvFrom comes after the chart secret: for duplicate keys Kubernetes
takes the LAST envFrom source, so these act as overrides. */}}
{{- $_ := set .Values "envFrom" (concat
(list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .))))
(.Values.extraEnvFrom | default (list))) }}
{{- include "yucca-common.deployment" . }}
@@ -0,0 +1,51 @@
{{- if .Values.migration.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "yucca-common.fullname" . }}-migrate
labels:
{{- include "yucca-common.labels" . | nindent 4 }}
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "10"
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
spec:
backoffLimit: 10
template:
metadata:
labels:
{{- include "yucca-common.selectorLabels" . | nindent 8 }}
job: migrate
spec:
restartPolicy: OnFailure
initContainers:
- name: wait-for-pg
image: postgres:18-alpine
command: ["sh", "-c"]
args:
- |
until pg_isready -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USERNAME"; do
echo "waiting for postgres..."; sleep 2;
done
env:
- { name: POSTGRES_HOST, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: host } } }
- { name: POSTGRES_PORT, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: port } } }
- { name: POSTGRES_USERNAME, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: username } } }
containers:
- name: migrate
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
workingDir: /app
command: ["sh", "-c"]
args:
- |
pnpm install --frozen-lockfile
cd packages/yucca-api
pnpm exec sql-tools -u "postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}" migrate
env:
- { name: POSTGRES_HOST, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: host } } }
- { name: POSTGRES_PORT, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: port } } }
- { name: POSTGRES_DATABASE, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: dbname } } }
- { name: POSTGRES_USERNAME, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: username } } }
- { name: POSTGRES_PASSWORD, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: password } } }
{{- end }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.secret" . }}
+1
View File
@@ -0,0 +1 @@
{{- include "yucca-common.service" . }}
+93
View File
@@ -0,0 +1,93 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name. This keeps
# service DNS identical whether rendered by Tilt (dev) or Flux (prod, per-app
# release names).
fullnameOverride: yucca-api
image:
repository: k3d-registry.localhost:5000/yucca-api
tag: dev
pullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3020
service:
type: ClusterIP
# CNPG-managed postgres Cluster name (see umbrella chart)
postgresClusterName: yucca-db
# OIDC provider in-cluster service
oidcIssuer: http://yucca-mock-oidc:8092
oidcRedirectUri: http://localhost:5173/api/auth/oidc/callback
oidcLogoutRedirectUri: http://localhost:5173
# DEV FIXTURES — not secrets. This is the project's well-known local-dev
# keypair (the same one committed in .mise/tasks/*/env); yucca-api signs
# device/restic JWTs with it and michael verifies with the matching public key.
# It must never protect anything real. Prod replaces this whole block with
# ExternalSecrets (1Password) — see kubernetes/README.md.
secretData:
JWT_PRIVATE_KEY: |
-----BEGIN PRIVATE KEY-----
MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgCla79+Sip4o2hZ1K
nQJYp2xU/nsCJmRoZmvXjeX6cW+hRANCAASkubBRIE7Sns/VS8ZF5XEVO8CwJ+AZ
Ls1C0ncCcmRhqKA7UxLknn0ji5FcKaku1zBOhxQYcxFVmsYtAxZ1ljgN
-----END PRIVATE KEY-----
OIDC_CLIENT_ID: "client ID"
OIDC_CLIENT_SECRET: "client secret"
# Extra envFrom sources appended AFTER the chart's own secret — for duplicate
# keys the last source wins, so this is the override hook. Tilt points it at
# the yucca-dev-env Secret (built from a gitignored root .env, op:// refs
# resolved via the 1Password CLI); prod can point it at an ExternalSecret.
# NB: explicit `env` entries below always beat envFrom — env vars the chart
# pins there (OIDC_ISSUER & co) are overridden via their Helm values instead.
extraEnvFrom: []
env:
- name: NODE_ENV
value: development
- name: YUCCA_API_PORT
value: "3020"
- name: LOG_LEVEL
value: debug
- name: OIDC_ALLOW_INSECURE
value: "true"
# Device-flow OIDC (required by yucca-api env schema; points at mock-oidc's
# registered device client).
- name: OIDC_DEVICE_ISSUER
value: http://yucca-mock-oidc:8092
- name: OIDC_DEVICE_CLIENT_ID
value: "device client ID"
- name: OIDC_DEVICE_ALLOW_INSECURE
value: "true"
- name: RESTIC_API_HOST
value: yucca-michael
- name: RESTIC_API_PORT
value: "3010"
- name: OTLP_METRICS_ENDPOINT
value: victoria-metrics:8428
- name: OTLP_METRICS_URL_PATH
value: /opentelemetry/v1/metrics
- name: OTLP_LOGS_ENDPOINT
value: victoria-logs:9428
- name: OTLP_LOGS_URL_PATH
value: /insert/opentelemetry/v1/logs
# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process
# still counts as Ready (this masked two real bugs). The startupProbe budgets
# for the dev watcher's first boot; no livenessProbe so dev never restart-loops.
startupProbe:
tcpSocket: { port: http }
periodSeconds: 5
failureThreshold: 60
readinessProbe:
tcpSocket: { port: http }
periodSeconds: 10
migration:
enabled: false
+5
View File
@@ -0,0 +1,5 @@
apiVersion: v2
name: yucca-common
description: Library chart with shared templates for Yucca services
type: library
version: 0.1.0
@@ -0,0 +1,70 @@
{{- define "yucca-common.deployment" -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "yucca-common.fullname" . }}
labels:
{{- include "yucca-common.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicas | default 1 }}
selector:
matchLabels:
{{- include "yucca-common.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "yucca-common.selectorLabels" . | nindent 8 }}
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.serviceAccountName }}
serviceAccountName: {{ . }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
{{- with .Values.command }}
command: {{ toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.args }}
args: {{ toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.workingDir }}
workingDir: {{ . }}
{{- end }}
ports:
{{- range .Values.ports }}
- name: {{ .name }}
containerPort: {{ .containerPort }}
protocol: {{ .protocol | default "TCP" }}
{{- end }}
{{- with .Values.env }}
env:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.envFrom }}
envFrom:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.startupProbe }}
startupProbe: {{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.livenessProbe }}
livenessProbe: {{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe: {{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.resources }}
resources: {{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumeMounts }}
volumeMounts: {{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.volumes }}
volumes: {{- toYaml . | nindent 8 }}
{{- end }}
{{- end -}}
@@ -0,0 +1,26 @@
{{/*
Fully qualified app name. Falls back to .Release.Name-.Chart.Name.
*/}}
{{- define "yucca-common.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- define "yucca-common.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "yucca-common.labels" -}}
app.kubernetes.io/name: {{ include "yucca-common.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
{{- end -}}
{{- define "yucca-common.selectorLabels" -}}
app.kubernetes.io/name: {{ include "yucca-common.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
+13
View File
@@ -0,0 +1,13 @@
{{- define "yucca-common.secret" -}}
{{- if .Values.secretData -}}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "yucca-common.fullname" . }}
labels:
{{- include "yucca-common.labels" . | nindent 4 }}
type: Opaque
stringData:
{{- toYaml .Values.secretData | nindent 2 }}
{{- end -}}
{{- end -}}
@@ -0,0 +1,19 @@
{{- define "yucca-common.service" -}}
apiVersion: v1
kind: Service
metadata:
name: {{ include "yucca-common.fullname" . }}
labels:
{{- include "yucca-common.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type | default "ClusterIP" }}
selector:
{{- include "yucca-common.selectorLabels" . | nindent 4 }}
ports:
{{- range .Values.ports }}
- name: {{ .name }}
port: {{ .servicePort | default .containerPort }}
targetPort: {{ .name }}
protocol: {{ .protocol | default "TCP" }}
{{- end }}
{{- end -}}
+27
View File
@@ -0,0 +1,27 @@
apiVersion: k3d.io/v1alpha5
kind: Simple
metadata:
name: yucca
servers: 1
agents: 0
image: rancher/k3s:v1.32.2-k3s1
registries:
create:
name: k3d-registry.localhost
host: '0.0.0.0'
hostPort: '5000'
options:
k3d:
wait: true
timeout: '60s'
k3s:
extraArgs:
- arg: '--disable=traefik'
nodeFilters: ['server:*']
- arg: '--disable=servicelb'
nodeFilters: ['server:*']
- arg: '--disable=metrics-server'
nodeFilters: ['server:*']
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
+140
View File
@@ -0,0 +1,140 @@
# Kubernetes (Flux GitOps)
Flux GitOps surface for the Yucca cluster, laid out in the
[home-operations](https://github.com/onedr0p/cluster-template) convention:
```
kubernetes/
├── bootstrap/ # one-time Flux install notes for a fresh cluster
├── flux/ # Flux sources (GitRepository/HelmRepository) + cluster entrypoint
├── components/ # reusable Kustomize components (cross-app concerns)
└── apps/ # applications, grouped by namespace
├── cnpg-system/ # CloudNativePG operator
├── rook-ceph/ # Rook-Ceph operator + dev cluster (S3 object store)
└── yucca/ # the product stack + its dev infra
```
## End-to-end tests against the local k3d stack
The e2e suites (`packages/e2e` + the web Playwright test) are written for a
host-local environment, while this stack runs in k3d. One command bridges them:
```bash
mise k3d:up && mise tilt:up # stack healthy (context k3d-yucca)
mise test:e2e:k3d # runs all e2e against the cluster
```
`mise test:e2e:k3d` (see `packages/e2e/k3d/run.sh`):
- **orchestration-api runs as a separate local process** (`:22676`) — it is
intentionally _not_ deployed to k8s; it targets the port-forwarded web.
- port-forwards the cluster services to the host ports the suites expect
(michael `:3010`, yucca-api `:3020`, mock-oidc `:8092`, web `:36033` + `:5173`).
- resolves the in-cluster OIDC issuer host (`yucca-mock-oidc`) on the host with
a Node DNS preload (jest) and Chromium `--host-resolver-rules` (Playwright) —
no `/etc/hosts`/sudo needed.
- sets `RESTIC_ENDPOINT=localhost:3010` on yucca-api **for the test run only**
(restic runs on the host; the chart keeps the in-cluster `yucca-michael`),
reverted on exit.
Note: michael creates **one S3 bucket per restic repository** (the bucket name
comes from the client JWT's `repository` claim; the S3 credentials are a static
RGW user). That's why it uses a full `CephObjectStoreUser` (charts/ceph-objectuser)
rather than a bucket-scoped ObjectBucketClaim.
## How it reconciles
Flux applies `kubernetes/flux/cluster` first (`cluster-repos` → `cluster-apps`).
`cluster-apps` builds `kubernetes/apps`, which aggregates one Flux `Kustomization`
(`ks.yaml`) per app. Each `ks.yaml` reconciles its `app/` directory, whose
`kustomization.yaml` applies a single `HelmRelease`. Ordering is expressed with
`dependsOn` (operator → database → apps).
```
apps/yucca/<app>/
├── ks.yaml # Flux Kustomization → ./app (+ dependsOn)
└── app/
├── kustomization.yaml
└── helmrelease.yaml # chart ref + values
```
## Single source of truth: this tree
The [Tiltfile](../Tiltfile) derives **everything it deploys** from the
HelmReleases here — first-party apps _and_ the remote-chart operators:
- First-party `HelmRelease`s reference the in-repo Helm charts via the `yucca`
`GitRepository` source (`chart: charts/<svc>`), so **no OCI publishing is
required**. Tilt renders the same charts with their dev defaults and injects
the locally-built, live-updated images.
- Remote `HelmRelease`s (cnpg, rook, victoria-\*) pin a chart version + values;
Tilt installs **exactly those**, from the `HelmRepository` sources declared in
`flux/repos/`. Bump a version or value once, in the HelmRelease — there is no
second copy to drift.
Service names are pinned with `fullnameOverride` in each chart's `values.yaml`,
so in-cluster DNS is identical whether a chart is rendered by Tilt (release
`yucca`) or by Flux (per-app release names).
| Layer | Reconciler | Image source | First-party values |
| -------- | ---------- | ------------------------------------------- | ------------------------------------------ |
| **Dev** | Tilt | `docker_build` → k3d registry, live-updated | chart defaults (`values.yaml`) |
| **Prod** | Flux | `ghcr.io/...` (per `HelmRelease`) | chart defaults + `HelmRelease.spec.values` |
## Dev vs prod
This tree currently mirrors the **dev** stack so it stays 1:1 with Tilt. Items
marked `TODO(prod)` (image registries, real OIDC/S3 endpoints, ingress, probes,
persistence, secrets) are where a future prod cluster overlay diverges. Notably:
- `mock-oidc` and `rook-ceph` are **dev-only**. Prod swaps in a real IdP, and
prod object storage is a **completely separate** Ceph (the bare-metal cluster
in [`ansible/ceph`](../ansible/ceph) / [`tf/`](../tf)) — not this Rook cluster.
- The Rook-Ceph dev cluster is single-node/single-replica and synthesizes a
loopback block device (k3d has no spare disk). See
[`charts/rook-ceph-cluster`](../charts/rook-ceph-cluster). `michael`'s S3
credentials come from a full RGW user
([`charts/ceph-objectuser`](../charts/ceph-objectuser)) whose Secret Rook
writes into the `yucca` namespace.
- The dev keypair/secrets committed in chart `secretData` are **well-known
fixtures** (the same keypair lives in `.mise/tasks/*/env`); they must become
`ExternalSecret`s backed by the org's 1Password (External Secrets Operator)
before prod.
## Real OIDC credentials in dev (`.env` + 1Password)
The k3d stack runs against mock-oidc out of the box. To point `yucca-api` at a
real IdP, drop a (gitignored) `.env` at the repo root — values may be
1Password `op://` references, resolved through the `op` CLI when the Tiltfile
loads:
```bash
OP_ACCOUNT="team-futo.1password.com" # only needed with multiple 1P accounts
OIDC_ISSUER="https://external-dev-gkhk8b.us1.zitadel.cloud"
OIDC_CLIENT_ID="op://yucca_tf_dev/CUSTOMER_ZITADEL_OAUTH_CLIENT_ID_DEV_TEST/password"
OIDC_CLIENT_SECRET="op://yucca_tf_dev/CUSTOMER_ZITADEL_OAUTH_CLIENT_SECRET_DEV_TEST/password"
```
Tilt turns the resolved pairs into the `yucca-dev-env` Secret and layers it
onto `yucca-api` as its last `envFrom` source (last source wins), so any key
here overrides the committed dev fixtures. `OIDC_ISSUER`/`OIDC_REDIRECT_URI`/
`OIDC_LOGOUT_REDIRECT_URI` are pinned by the chart as explicit env (which
beats `envFrom`) and are mapped onto their Helm values instead — keep those
three non-secret, as Helm flags are visible in the Tilt UI. Editing or
deleting `.env` redeploys automatically; without it (CI, fresh clones)
nothing changes.
Caveats: the IdP must allow `http://localhost:5173/api/auth/oidc/callback` as
a redirect URI; the device flow (`OIDC_DEVICE_*`) stays on mock-oidc; and the
web e2e suite logs in via mock-oidc, so remove `.env` before
`mise test:e2e:k3d`.
## Validate locally
```bash
# render the kustomize graph
kubectl kustomize kubernetes/apps
# build the whole tree (Kustomizations + HelmReleases) the way Flux would
# (https://github.com/allenporter/flux-local)
flux-local build all kubernetes --enable-helm --no-enable-dns
```
@@ -0,0 +1,24 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cloudnative-pg
namespace: cnpg-system
spec:
interval: 1h
chart:
spec:
chart: cloudnative-pg
version: 0.23.0
sourceRef:
kind: HelmRepository
name: cloudnative-pg
namespace: flux-system
install:
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,16 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cnpg-operator
namespace: flux-system
spec:
targetNamespace: cnpg-system
path: ./kubernetes/apps/cnpg-system/cloudnative-pg/app
prune: true
wait: true
interval: 1h
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./namespace.yaml
- ./cloudnative-pg/ks.yaml
@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: cnpg-system
+6
View File
@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./cnpg-system
- ./rook-ceph
- ./yucca
@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./namespace.yaml
- ./rook-ceph-operator/ks.yaml
- ./rook-ceph-cluster/ks.yaml
+5
View File
@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: rook-ceph
@@ -0,0 +1,24 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: rook-ceph-cluster
namespace: rook-ceph
spec:
interval: 1h
chart:
spec:
chart: charts/rook-ceph-cluster
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
# Dev defaults (single-node, single-replica, loopback OSD) live in the chart's
# values.yaml. This is DEV ONLY — prod object storage is a separate Ceph.
values: {}
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,18 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: rook-ceph-cluster
namespace: flux-system
spec:
targetNamespace: rook-ceph
path: ./kubernetes/apps/rook-ceph/rook-ceph-cluster/app
prune: true
wait: true
interval: 1h
timeout: 15m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: rook-ceph-operator
@@ -0,0 +1,33 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: rook-ceph-operator
namespace: rook-ceph
spec:
interval: 1h
chart:
spec:
chart: rook-ceph
version: v1.20.0
sourceRef:
kind: HelmRepository
name: rook-release
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values:
# DEV footprint: object store only needs the bucket provisioner, not CSI.
csi:
enableRbdDriver: false
enableCephfsDriver: false
enableDiscoveryDaemon: true
monitoring:
enabled: false
# The dev cluster's OSD runs on a loopback block device (k3d has no spare
# disk); Rook filters loop devices out of discovery unless this is set.
allowLoopDevices: true
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,16 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: rook-ceph-operator
namespace: flux-system
spec:
targetNamespace: rook-ceph
path: ./kubernetes/apps/rook-ceph/rook-ceph-operator/app
prune: true
wait: true
interval: 1h
timeout: 10m
sourceRef:
kind: GitRepository
name: yucca
@@ -0,0 +1,24 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-database
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/cnpg-cluster
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
# Dev-mirror posture: chart defaults (1 instance, 1Gi) keep this tree 1:1
# with the Tilt deployment. TODO(prod): a prod overlay raises instances/storage.
values: {}
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
+22
View File
@@ -0,0 +1,22 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-database
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-database
path: ./kubernetes/apps/yucca/database/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: cnpg-operator
+13
View File
@@ -0,0 +1,13 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./namespace.yaml
- ./database/ks.yaml
- ./object-user/ks.yaml
- ./mock-oidc/ks.yaml
- ./victoria-metrics/ks.yaml
- ./victoria-logs/ks.yaml
- ./michael/ks.yaml
- ./yucca-api/ks.yaml
- ./yucca-admin-api/ks.yaml
- ./web/ks.yaml
@@ -0,0 +1,28 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-michael
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/michael
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
values:
# Dev (Tilt) uses the chart defaults; prod overrides the image + secrets.
image:
repository: ghcr.io/immich-app/yucca/michael # TODO: confirm prod registry
tag: 0.0.1
# TODO(prod): source JWT_SECRET / S3 creds from an ExternalSecret, not values
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
+22
View File
@@ -0,0 +1,22 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-michael
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-michael
path: ./kubernetes/apps/yucca/michael/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: yucca-object-user
@@ -0,0 +1,28 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-mock-oidc
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/mock-oidc
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values:
# DEV ONLY. Prod uses a real IdP (e.g. Zitadel) — drop this release and
# point yucca-api/admin-api OIDC_* at the real issuer via ExternalSecrets.
fullnameOverride: yucca-mock-oidc
image:
repository: ghcr.io/immich-app/yucca/mock-oidc-provider # TODO: confirm prod registry
tag: 0.0.1
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
+20
View File
@@ -0,0 +1,20 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-mock-oidc
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-mock-oidc
path: ./kubernetes/apps/yucca/mock-oidc/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
+5
View File
@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: yucca
@@ -0,0 +1,22 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-object-user
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/ceph-objectuser
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values: {}
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
+22
View File
@@ -0,0 +1,22 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-object-user
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-object-user
path: ./kubernetes/apps/yucca/object-user/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: rook-ceph-cluster
@@ -0,0 +1,29 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-victoria-logs
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: victoria-logs-single
version: 0.11.32
sourceRef:
kind: HelmRepository
name: victoriametrics
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values:
server:
fullnameOverride: victoria-logs
# Dev-mirror posture (Tilt installs these exact values).
# TODO(prod): enable + size persistence for the target cluster.
persistentVolume:
enabled: false
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,20 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-victoria-logs
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-victoria-logs
path: ./kubernetes/apps/yucca/victoria-logs/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
@@ -0,0 +1,29 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-victoria-metrics
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: victoria-metrics-single
version: 0.35.0
sourceRef:
kind: HelmRepository
name: victoriametrics
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values:
server:
fullnameOverride: victoria-metrics
# Dev-mirror posture (Tilt installs these exact values).
# TODO(prod): enable + size persistence for the target cluster.
persistentVolume:
enabled: false
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,20 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-victoria-metrics
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-victoria-metrics
path: ./kubernetes/apps/yucca/victoria-metrics/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
@@ -0,0 +1,28 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-web
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/web
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
values:
image:
repository: ghcr.io/immich-app/yucca/web # TODO: confirm prod registry
tag: 0.0.1
# YUCCA_API_URL/PUBLIC_API_URL default to the in-cluster yucca-api service,
# which is correct in prod too. TODO(prod): add ingress for external access.
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
+22
View File
@@ -0,0 +1,22 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-web
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-web
path: ./kubernetes/apps/yucca/web/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: yucca-api
@@ -0,0 +1,27 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-admin-api
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/yucca-admin-api
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
values:
image:
repository: ghcr.io/immich-app/yucca/yucca-admin-api # TODO: confirm prod registry
tag: 0.0.1
# TODO(prod): real OIDC_ADMIN_* URLs + secrets via ExternalSecret
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,23 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-admin-api
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-admin-api
path: ./kubernetes/apps/yucca/yucca-admin-api/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: yucca-database
- name: yucca-mock-oidc

Some files were not shown because too many files have changed in this diff Show More