From 070e22a7bb16fdfe506fa334547e173982a1462b Mon Sep 17 00:00:00 2001 From: Antoine Lecompte <38678863+nutgood@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:17:37 -0400 Subject: [PATCH] feat: local k8s (#85) * impl. local kube * add support for op injected oidc secrets * ci: set least-privilege workflow token permissions --- .dockerignore | 9 + .github/workflows/ci.yml | 98 +++-- .gitignore | 10 + .mise/config.toml | 18 +- .mise/tasks/k3d/down | 15 + .mise/tasks/k3d/reset | 4 + .mise/tasks/k3d/up | 16 + .mise/tasks/k8s/validate | 45 +++ .mise/tasks/test/e2e/k3d | 5 + .mise/tasks/test/e2e/wait | 14 +- .mise/tasks/test/integration/k3d | 66 +++ .mise/tasks/tilt/ci | 11 + .mise/tasks/tilt/ci-infra | 19 + .mise/tasks/tilt/down | 5 + .mise/tasks/tilt/up | 10 + .prettierignore | 5 + README.md | 36 +- Tiltfile | 377 ++++++++++++++++++ charts/ceph-objectuser/Chart.yaml | 9 + .../ceph-objectuser/templates/objectuser.yaml | 8 + charts/ceph-objectuser/values.yaml | 7 + charts/cnpg-cluster/Chart.yaml | 6 + charts/cnpg-cluster/templates/cluster.yaml | 12 + charts/cnpg-cluster/values.yaml | 8 + charts/michael/Chart.yaml | 10 + charts/michael/templates/deployment.yaml | 1 + charts/michael/templates/secret.yaml | 1 + charts/michael/templates/service.yaml | 1 + charts/michael/values.yaml | 76 ++++ charts/mock-oidc/Chart.yaml | 10 + charts/mock-oidc/templates/deployment.yaml | 1 + charts/mock-oidc/templates/service.yaml | 1 + charts/mock-oidc/values.yaml | 53 +++ charts/rook-ceph-cluster/Chart.yaml | 9 + .../templates/cephcluster.yaml | 46 +++ .../templates/cephobjectstore.yaml | 25 ++ .../templates/loop-device.yaml | 73 ++++ charts/rook-ceph-cluster/values.yaml | 44 ++ charts/web/Chart.yaml | 10 + charts/web/templates/deployment.yaml | 1 + charts/web/templates/service.yaml | 1 + charts/web/values.yaml | 38 ++ charts/yucca-admin-api/Chart.yaml | 10 + .../yucca-admin-api/templates/deployment.yaml | 12 + charts/yucca-admin-api/templates/secret.yaml | 1 + charts/yucca-admin-api/templates/service.yaml | 1 + charts/yucca-admin-api/values.yaml | 58 +++ charts/yucca-api/Chart.yaml | 10 + charts/yucca-api/templates/deployment.yaml | 16 + charts/yucca-api/templates/migration-job.yaml | 51 +++ charts/yucca-api/templates/secret.yaml | 1 + charts/yucca-api/templates/service.yaml | 1 + charts/yucca-api/values.yaml | 93 +++++ charts/yucca-common/Chart.yaml | 5 + charts/yucca-common/templates/_deployment.tpl | 70 ++++ charts/yucca-common/templates/_helpers.tpl | 26 ++ charts/yucca-common/templates/_secret.tpl | 13 + charts/yucca-common/templates/_service.tpl | 19 + k3d.yaml | 27 ++ kubernetes/README.md | 140 +++++++ .../cloudnative-pg/app/helmrelease.yaml | 24 ++ .../cloudnative-pg/app/kustomization.yaml | 4 + .../apps/cnpg-system/cloudnative-pg/ks.yaml | 16 + .../apps/cnpg-system/kustomization.yaml | 5 + kubernetes/apps/cnpg-system/namespace.yaml | 5 + kubernetes/apps/kustomization.yaml | 6 + kubernetes/apps/rook-ceph/kustomization.yaml | 6 + kubernetes/apps/rook-ceph/namespace.yaml | 5 + .../rook-ceph-cluster/app/helmrelease.yaml | 24 ++ .../rook-ceph-cluster/app/kustomization.yaml | 4 + .../apps/rook-ceph/rook-ceph-cluster/ks.yaml | 18 + .../rook-ceph-operator/app/helmrelease.yaml | 33 ++ .../rook-ceph-operator/app/kustomization.yaml | 4 + .../apps/rook-ceph/rook-ceph-operator/ks.yaml | 16 + .../apps/yucca/database/app/helmrelease.yaml | 24 ++ .../yucca/database/app/kustomization.yaml | 4 + kubernetes/apps/yucca/database/ks.yaml | 22 + kubernetes/apps/yucca/kustomization.yaml | 13 + .../apps/yucca/michael/app/helmrelease.yaml | 28 ++ .../apps/yucca/michael/app/kustomization.yaml | 4 + kubernetes/apps/yucca/michael/ks.yaml | 22 + .../apps/yucca/mock-oidc/app/helmrelease.yaml | 28 ++ .../yucca/mock-oidc/app/kustomization.yaml | 4 + kubernetes/apps/yucca/mock-oidc/ks.yaml | 20 + kubernetes/apps/yucca/namespace.yaml | 5 + .../yucca/object-user/app/helmrelease.yaml | 22 + .../yucca/object-user/app/kustomization.yaml | 4 + kubernetes/apps/yucca/object-user/ks.yaml | 22 + .../yucca/victoria-logs/app/helmrelease.yaml | 29 ++ .../victoria-logs/app/kustomization.yaml | 4 + kubernetes/apps/yucca/victoria-logs/ks.yaml | 20 + .../victoria-metrics/app/helmrelease.yaml | 29 ++ .../victoria-metrics/app/kustomization.yaml | 4 + .../apps/yucca/victoria-metrics/ks.yaml | 20 + .../apps/yucca/web/app/helmrelease.yaml | 28 ++ .../apps/yucca/web/app/kustomization.yaml | 4 + kubernetes/apps/yucca/web/ks.yaml | 22 + .../yucca-admin-api/app/helmrelease.yaml | 27 ++ .../yucca-admin-api/app/kustomization.yaml | 4 + kubernetes/apps/yucca/yucca-admin-api/ks.yaml | 23 ++ .../apps/yucca/yucca-api/app/helmrelease.yaml | 34 ++ .../yucca/yucca-api/app/kustomization.yaml | 4 + kubernetes/apps/yucca/yucca-api/ks.yaml | 24 ++ kubernetes/bootstrap/README.md | 61 +++ .../components/common/kustomization.yaml | 6 + kubernetes/flux/cluster/apps.yaml | 16 + kubernetes/flux/cluster/kustomization.yaml | 5 + kubernetes/flux/cluster/repos.yaml | 14 + kubernetes/flux/repos/git-yucca.yaml | 20 + .../flux/repos/helm-cloudnative-pg.yaml | 9 + kubernetes/flux/repos/helm-rook.yaml | 9 + .../flux/repos/helm-victoriametrics.yaml | 9 + kubernetes/flux/repos/kustomization.yaml | 7 + packages/e2e/k3d/hostmap.cjs | 28 ++ packages/e2e/k3d/run.sh | 77 ++++ packages/michael/.air.toml | 9 + packages/michael/Dockerfile | 17 +- packages/web/Dockerfile | 42 +- packages/web/playwright.k3d.config.ts | 15 + packages/web/vite.config.ts | 4 +- packages/yucca-admin-api/Dockerfile | 45 ++- packages/yucca-api/Dockerfile | 37 +- 122 files changed, 2814 insertions(+), 52 deletions(-) create mode 100755 .mise/tasks/k3d/down create mode 100755 .mise/tasks/k3d/reset create mode 100755 .mise/tasks/k3d/up create mode 100755 .mise/tasks/k8s/validate create mode 100755 .mise/tasks/test/e2e/k3d create mode 100755 .mise/tasks/test/integration/k3d create mode 100755 .mise/tasks/tilt/ci create mode 100755 .mise/tasks/tilt/ci-infra create mode 100755 .mise/tasks/tilt/down create mode 100755 .mise/tasks/tilt/up create mode 100644 Tiltfile create mode 100644 charts/ceph-objectuser/Chart.yaml create mode 100644 charts/ceph-objectuser/templates/objectuser.yaml create mode 100644 charts/ceph-objectuser/values.yaml create mode 100644 charts/cnpg-cluster/Chart.yaml create mode 100644 charts/cnpg-cluster/templates/cluster.yaml create mode 100644 charts/cnpg-cluster/values.yaml create mode 100644 charts/michael/Chart.yaml create mode 100644 charts/michael/templates/deployment.yaml create mode 100644 charts/michael/templates/secret.yaml create mode 100644 charts/michael/templates/service.yaml create mode 100644 charts/michael/values.yaml create mode 100644 charts/mock-oidc/Chart.yaml create mode 100644 charts/mock-oidc/templates/deployment.yaml create mode 100644 charts/mock-oidc/templates/service.yaml create mode 100644 charts/mock-oidc/values.yaml create mode 100644 charts/rook-ceph-cluster/Chart.yaml create mode 100644 charts/rook-ceph-cluster/templates/cephcluster.yaml create mode 100644 charts/rook-ceph-cluster/templates/cephobjectstore.yaml create mode 100644 charts/rook-ceph-cluster/templates/loop-device.yaml create mode 100644 charts/rook-ceph-cluster/values.yaml create mode 100644 charts/web/Chart.yaml create mode 100644 charts/web/templates/deployment.yaml create mode 100644 charts/web/templates/service.yaml create mode 100644 charts/web/values.yaml create mode 100644 charts/yucca-admin-api/Chart.yaml create mode 100644 charts/yucca-admin-api/templates/deployment.yaml create mode 100644 charts/yucca-admin-api/templates/secret.yaml create mode 100644 charts/yucca-admin-api/templates/service.yaml create mode 100644 charts/yucca-admin-api/values.yaml create mode 100644 charts/yucca-api/Chart.yaml create mode 100644 charts/yucca-api/templates/deployment.yaml create mode 100644 charts/yucca-api/templates/migration-job.yaml create mode 100644 charts/yucca-api/templates/secret.yaml create mode 100644 charts/yucca-api/templates/service.yaml create mode 100644 charts/yucca-api/values.yaml create mode 100644 charts/yucca-common/Chart.yaml create mode 100644 charts/yucca-common/templates/_deployment.tpl create mode 100644 charts/yucca-common/templates/_helpers.tpl create mode 100644 charts/yucca-common/templates/_secret.tpl create mode 100644 charts/yucca-common/templates/_service.tpl create mode 100644 k3d.yaml create mode 100644 kubernetes/README.md create mode 100644 kubernetes/apps/cnpg-system/cloudnative-pg/app/helmrelease.yaml create mode 100644 kubernetes/apps/cnpg-system/cloudnative-pg/app/kustomization.yaml create mode 100644 kubernetes/apps/cnpg-system/cloudnative-pg/ks.yaml create mode 100644 kubernetes/apps/cnpg-system/kustomization.yaml create mode 100644 kubernetes/apps/cnpg-system/namespace.yaml create mode 100644 kubernetes/apps/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/namespace.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-cluster/app/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-cluster/app/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-cluster/ks.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-operator/app/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-operator/app/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph-operator/ks.yaml create mode 100644 kubernetes/apps/yucca/database/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/database/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/database/ks.yaml create mode 100644 kubernetes/apps/yucca/kustomization.yaml create mode 100644 kubernetes/apps/yucca/michael/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/michael/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/michael/ks.yaml create mode 100644 kubernetes/apps/yucca/mock-oidc/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/mock-oidc/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/mock-oidc/ks.yaml create mode 100644 kubernetes/apps/yucca/namespace.yaml create mode 100644 kubernetes/apps/yucca/object-user/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/object-user/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/object-user/ks.yaml create mode 100644 kubernetes/apps/yucca/victoria-logs/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/victoria-logs/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/victoria-logs/ks.yaml create mode 100644 kubernetes/apps/yucca/victoria-metrics/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/victoria-metrics/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/victoria-metrics/ks.yaml create mode 100644 kubernetes/apps/yucca/web/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/web/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/web/ks.yaml create mode 100644 kubernetes/apps/yucca/yucca-admin-api/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/yucca-admin-api/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/yucca-admin-api/ks.yaml create mode 100644 kubernetes/apps/yucca/yucca-api/app/helmrelease.yaml create mode 100644 kubernetes/apps/yucca/yucca-api/app/kustomization.yaml create mode 100644 kubernetes/apps/yucca/yucca-api/ks.yaml create mode 100644 kubernetes/bootstrap/README.md create mode 100644 kubernetes/components/common/kustomization.yaml create mode 100644 kubernetes/flux/cluster/apps.yaml create mode 100644 kubernetes/flux/cluster/kustomization.yaml create mode 100644 kubernetes/flux/cluster/repos.yaml create mode 100644 kubernetes/flux/repos/git-yucca.yaml create mode 100644 kubernetes/flux/repos/helm-cloudnative-pg.yaml create mode 100644 kubernetes/flux/repos/helm-rook.yaml create mode 100644 kubernetes/flux/repos/helm-victoriametrics.yaml create mode 100644 kubernetes/flux/repos/kustomization.yaml create mode 100644 packages/e2e/k3d/hostmap.cjs create mode 100755 packages/e2e/k3d/run.sh create mode 100644 packages/michael/.air.toml create mode 100644 packages/web/playwright.k3d.config.ts diff --git a/.dockerignore b/.dockerignore index aa5ed7ec..ba6bcfb9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,12 @@ e2e **/.env* Dockerfile packages/**/build +.dev +.mise/data +**/tmp +**/.turbo +**/.cache +charts +Tiltfile +k3d.yaml +compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a1e5626..1e9ddf9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,11 @@ on: push: branches: [main] +# Every job only reads the repo (checkout + tool downloads); nothing writes +# back through the token. +permissions: + contents: read + concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -20,14 +25,19 @@ jobs: - name: Setup Mise uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0 + env: + # mise downloads tools from GitHub releases; anonymous requests hit + # API rate limits with four parallel jobs. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: mise prepare - name: Run checks run: mise check - integration: - name: Integration Tests + k8s-validate: + name: Validate Kubernetes Surface runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: @@ -35,17 +45,53 @@ jobs: - name: Setup Mise uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0 + env: + # mise downloads tools from GitHub releases; anonymous requests hit + # API rate limits with four parallel jobs. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # helm template + kubeconform per chart, then flux-local builds the whole + # kubernetes/ tree the way Flux would. No cluster involved. + - name: Validate charts + Flux tree + run: mise run k8s:validate + + integration: + name: Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + with: + persist-credentials: false + + - name: Setup Mise + uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0 + env: + # mise downloads tools from GitHub releases; anonymous requests hit + # API rate limits with four parallel jobs. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: mise prepare - - name: Start services + # The stack (Ceph image, dev images, k3s) is heavy; the stock runner has + # ~14GB free, so drop the biggest unused toolchains up front. + - name: Free runner disk space + run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true + + # Tests boot the apps on the runner; only the infra (CNPG postgres, Ceph + # RGW, mock-oidc, victoria) comes from the cluster — so skip the four + # heavy app images entirely. + - name: Stand up k3d infra run: | - mise docker:start - - name: Run integration tests - run: mise test:integration + mise k3d:up + mise tilt:ci-infra + + - name: Run integration tests against the cluster + run: mise test:integration:k3d e2e: name: End-to-end Tests runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: @@ -53,28 +99,26 @@ jobs: - name: Setup Mise uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0 + env: + # mise downloads tools from GitHub releases; anonymous requests hit + # API rate limits with four parallel jobs. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: mise prepare - - name: Start services & run end-to-end tests + - name: Free runner disk space + run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true + + - name: Install Playwright browser + run: pnpm --filter web exec playwright install --with-deps chromium + + # Full stack on k3d, then the whole e2e surface — the jest suites + # (it-works, restic-api, yucca-api, orchestration-api) AND the web + # Playwright test — via the same runner developers use locally. + # orchestration-api runs as a separate host process by design. + - name: Stand up the full k3d stack run: | - mise docker:start - mise dev & - mise test:e2e + mise k3d:up + mise tilt:ci - e2e-web: - name: End-to-end (Web) Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - with: - persist-credentials: false - - - name: Setup Mise - uses: immich-app/devtools/actions/use-mise@cd24790a7f5f6439ac32cc94f5523cb2de8bfa8c # use-mise-action-v1.1.0 - - run: mise prepare - - - name: Start services & run end-to-end tests - run: | - mise docker:start - mise dev & - mise test:e2e:web + - name: Run end-to-end tests against the cluster + run: mise test:e2e:k3d diff --git a/.gitignore b/.gitignore index 969ae3b3..d2386647 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,16 @@ mise.local.toml dist/ packages/michael/michael +# k3d/Tilt dev stack — Helm builds subchart snapshots on the fly; the .dev/ +# directory holds persistent service data carried over from the compose flow. +.dev/ +charts/**/charts/ +charts/**/tmpcharts-*/ +charts/**/Chart.lock + +# air (Go live-reload) temp build output in michael package +packages/michael/tmp/ + # OpenTofu / Terraform # .terraform.lock.hcl IS committed for reproducibility **/.terraform/ diff --git a/.mise/config.toml b/.mise/config.toml index 42d3a4a4..26143b7d 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -7,6 +7,20 @@ restic = "0.18.0" gh = "2.25.0" "github:git-town/git-town" = "22.4.0" +k3d = "5.8.3" +kubectl = "1.32.2" +helm = "3.17.0" +tilt = "0.34.5" + +# Static validation of the k8s surface (mise run k8s:validate, used by CI). +# flux-local runs via `uvx` (see .mise/tasks/k8s/validate): uv auto-fetches a +# Python matching its requires-python, so the host's Python version (3.12 on +# GitHub runners, 3.14 on dev machines) doesn't matter. +kubeconform = "0.8.0" +kustomize = "5.8.1" # flux-local shells out to it +flux2 = "2.7.5" # ...and to the flux CLI +uv = "0.9.18" + # Infrastructure tooling (added for tf/ and ansible/ subtrees) opentofu = "1.11.5" terragrunt = "0.99.4" @@ -30,7 +44,9 @@ depends = ["*:test"] [tasks."test:integration"] description = "Run all integration tests" -run = "mise run '*:test:integration' --jobs 1" +# NB: mise flags must precede the task pattern — anything after it is forwarded +# to the tasks themselves (jest/go test choke on a stray --jobs). +run = "mise run --jobs 1 '*:test:integration'" [tasks."test:e2e:web"] description = "Run all web e2e tests" diff --git a/.mise/tasks/k3d/down b/.mise/tasks/k3d/down new file mode 100755 index 00000000..77d24e05 --- /dev/null +++ b/.mise/tasks/k3d/down @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +#MISE description="Delete k3d cluster" +set -euo pipefail + +CLUSTER_NAME="yucca" + +if k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then + k3d cluster delete "${CLUSTER_NAME}" +else + echo "Cluster ${CLUSTER_NAME} does not exist" +fi + +if k3d registry get k3d-registry.localhost >/dev/null 2>&1; then + k3d registry delete k3d-registry.localhost +fi diff --git a/.mise/tasks/k3d/reset b/.mise/tasks/k3d/reset new file mode 100755 index 00000000..9c829274 --- /dev/null +++ b/.mise/tasks/k3d/reset @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Recreate k3d cluster" +#MISE depends=["k3d:down", "k3d:up"] +set -euo pipefail diff --git a/.mise/tasks/k3d/up b/.mise/tasks/k3d/up new file mode 100755 index 00000000..ba68c7ff --- /dev/null +++ b/.mise/tasks/k3d/up @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#MISE description="Create k3d cluster for local dev" +set -euo pipefail + +CLUSTER_NAME="yucca" + +if k3d cluster get "${CLUSTER_NAME}" >/dev/null 2>&1; then + echo "Cluster ${CLUSTER_NAME} already exists" +else + k3d cluster create --config k3d.yaml +fi + +kubectl wait --for=condition=Ready nodes --all --timeout=120s +echo "" +echo "Cluster ready. Context: k3d-${CLUSTER_NAME}" +echo "Registry: k3d-registry.localhost:5000" diff --git a/.mise/tasks/k8s/validate b/.mise/tasks/k8s/validate new file mode 100755 index 00000000..78655b39 --- /dev/null +++ b/.mise/tasks/k8s/validate @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +#MISE description="Statically validate the k8s surface (charts + Flux tree)" +#MISE dir="{{config_root}}" +# Renders every chart (helm template + kubeconform) and builds the whole Flux +# tree the way Flux would (flux-local --enable-helm). No cluster needed; this +# is the CI gate for kubernetes/ and charts/ changes. +# +# NB: don't run this while Tilt is converging — Tilt's helm-deps resource +# rebuilds charts/*/charts/ (rm -rf + dependency build) and races the renders. +set -euo pipefail + +LIB_CONSUMERS=(yucca-api yucca-admin-api web michael mock-oidc) +ALL_CHARTS=(yucca-api yucca-admin-api web michael mock-oidc cnpg-cluster ceph-objectuser rook-ceph-cluster) + +echo "==> helm dependency build (yucca-common consumers)" +for c in "${LIB_CONSUMERS[@]}"; do + (cd "charts/$c" && helm dependency build >/dev/null) +done + +echo "==> helm template + kubeconform" +for c in "${ALL_CHARTS[@]}"; do + ns=yucca + [ "$c" = "rook-ceph-cluster" ] && ns=rook-ceph + # NB: release name must not be YAML-boolean-ish ("y"/"on"/...): it lands in + # labels and kubeconform reads it back as a bool. + helm template yucca "charts/$c" -n "$ns" \ + | kubeconform -strict -ignore-missing-schemas - \ + && echo " OK $c" +done + +echo "==> kustomize build (Flux entrypoints)" +kustomize build kubernetes/apps >/dev/null && echo " OK kubernetes/apps" +kustomize build kubernetes/flux/repos >/dev/null && echo " OK kubernetes/flux/repos" +kustomize build kubernetes/flux/cluster >/dev/null && echo " OK kubernetes/flux/cluster" + +echo "==> flux-local build (full tree, helm rendering as Flux would)" +# Via uvx so uv supplies a Python matching flux-local's requires-python +# (>=3.13) regardless of the host. One retry: flux-local fans out `flux build +# ks` subprocesses which very rarely segfault under load. +flux_local() { uvx --from "flux-local==8.2.0" flux-local "$@"; } +flux_local build all kubernetes --enable-helm --no-enable-dns >/dev/null \ + || flux_local build all kubernetes --enable-helm --no-enable-dns >/dev/null +echo " OK flux-local" + +echo "k8s surface: ALL VALID" diff --git a/.mise/tasks/test/e2e/k3d b/.mise/tasks/test/e2e/k3d new file mode 100755 index 00000000..735f3df9 --- /dev/null +++ b/.mise/tasks/test/e2e/k3d @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#MISE description="Run all e2e tests against the local k3d/Tilt stack (orchestration-api runs separately)" +#MISE dir="{{config_root}}" +set -euo pipefail +exec ./packages/e2e/k3d/run.sh diff --git a/.mise/tasks/test/e2e/wait b/.mise/tasks/test/e2e/wait index b83ed58f..572077dc 100755 --- a/.mise/tasks/test/e2e/wait +++ b/.mise/tasks/test/e2e/wait @@ -1,5 +1,7 @@ #!/usr/bin/env bash #MISE description="Wait for development environment to be ready" +# Targets the compose/`mise dev` flow (the k3d flow has its own readiness +# handling in packages/e2e/k3d/run.sh and does not use this task). set -e source "$(dirname "$0")/../../restic-api/env" source "$(dirname "$0")/../../yucca-api/env" @@ -9,13 +11,15 @@ echo Ensure dev environment is running before invoking tests. wait_for_port() { local port="$1" local name="$2" - local deadline=$(( $(date +%s) + 30 )) + local timeout="${3:-60}" + local elapsed=0 while ! nc -z localhost "$port" 2>/dev/null; do - if (( $(date +%s) >= deadline )); then - echo "Timed out waiting for $name (:$port)" >&2 - exit 1 + if [ "$elapsed" -ge "$timeout" ]; then + echo "timed out waiting for $name (localhost:$port) after ${timeout}s" >&2 + return 1 fi sleep 0.5 + elapsed=$((elapsed + 1)) done } @@ -23,4 +27,4 @@ wait_for_port 8092 mock-oidc-provider wait_for_port "$RESTIC_API_PORT" restic-api wait_for_port "$YUCCA_API_PORT" yucca-api wait_for_port 22676 orchestration-api -wait_for_port 36033 web +wait_for_port "${WEB_PORT:-36033}" web diff --git a/.mise/tasks/test/integration/k3d b/.mise/tasks/test/integration/k3d new file mode 100755 index 00000000..4463c7b1 --- /dev/null +++ b/.mise/tasks/test/integration/k3d @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +#MISE description="Run all integration tests against the local k3d stack's infra" +#MISE dir="{{config_root}}" +# Kube replacement for the compose-backed `mise docker:start && mise test:integration`: +# the tests boot the apps on this host but take their infrastructure — postgres +# (CNPG), S3 (Ceph RGW), the OIDC provider and the victoria pair — from the k3d +# cluster, port-forwarded to the same localhost ports the compose flow used. +# +# Prereq: mise k3d:up && mise tilt:ci-infra (apps not required) +set -euo pipefail + +[ "$(kubectl config current-context)" = "k3d-yucca" ] || { + echo "Not on k3d-yucca. Run: mise k3d:up && mise tilt:ci-infra" >&2; exit 1; } + +PF_PIDS=() +cleanup() { + for p in "${PF_PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done +} +trap cleanup EXIT + +wait_for() { + local desc="$1"; shift + for _ in $(seq 1 60); do "$@" >/dev/null 2>&1 && return 0; sleep 5; done + echo "timed out waiting for $desc" >&2; return 1 +} + +echo "==> wait for infra (tilt ci-infra returns before Rook mints the S3 user)" +wait_for "CNPG cluster Ready" kubectl -n yucca wait --for=condition=Ready cluster/yucca-db --timeout=5s +wait_for "Ceph S3 user secret" kubectl -n yucca get secret rook-ceph-object-user-yucca-michael +wait_for "mock-oidc Available" kubectl -n yucca wait --for=condition=Available deploy/yucca-mock-oidc --timeout=5s + +echo "==> port-forward infra to the localhost ports the tests expect" +kubectl port-forward -n yucca svc/yucca-db-rw 15432:5432 >/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n rook-ceph svc/rook-ceph-rgw-yucca 9000:80 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n yucca svc/yucca-mock-oidc 8092:8092 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n yucca svc/victoria-metrics 8428:8428 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n yucca svc/victoria-logs 9428:9428 >>/tmp/yucca-int-pf.log 2>&1 & PF_PIDS+=($!) +probe() { (exec 3<>"/dev/tcp/localhost/$1") 2>/dev/null; } +for port in 15432 9000 8092; do + wait_for "localhost:$port" probe "$port" +done + +echo "==> export cluster credentials (the per-package env files only set defaults)" +POSTGRES_HOST=localhost +POSTGRES_PORT=15432 +POSTGRES_USERNAME="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.username}' | base64 -d)" +POSTGRES_PASSWORD="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.password}' | base64 -d)" +POSTGRES_DATABASE="$(kubectl -n yucca get secret yucca-db-app -o jsonpath='{.data.dbname}' | base64 -d)" +export POSTGRES_HOST POSTGRES_PORT POSTGRES_USERNAME POSTGRES_PASSWORD POSTGRES_DATABASE + +S3_ENDPOINT=http://localhost:9000 +S3_ACCESS_KEY_ID="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-michael -o jsonpath='{.data.AccessKey}' | base64 -d)" +S3_SECRET_ACCESS_KEY="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-michael -o jsonpath='{.data.SecretKey}' | base64 -d)" +S3_REGION=us-east-1 +S3_FORCE_PATH_STYLE=true +export S3_ENDPOINT S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY S3_REGION S3_FORCE_PATH_STYLE + +# The deployed mock-oidc advertises its in-cluster name as the issuer; the dns +# preload resolves it to the port-forward for every node process (jest + the +# apps it boots). Same split-horizon glue as the e2e runner. +export OIDC_ISSUER=http://yucca-mock-oidc:8092 +export OIDC_DEVICE_ISSUER=http://yucca-mock-oidc:8092 +export NODE_OPTIONS="--require $PWD/packages/e2e/k3d/hostmap.cjs${NODE_OPTIONS:+ ${NODE_OPTIONS}}" + +echo "==> run the integration suites" +mise run test:integration diff --git a/.mise/tasks/tilt/ci b/.mise/tasks/tilt/ci new file mode 100755 index 00000000..db5fe4d9 --- /dev/null +++ b/.mise/tasks/tilt/ci @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +#MISE description="Build + deploy + wait for the k3d stack (tilt ci); optional resource subset" +#MISE dir="{{config_root}}" +set -euo pipefail + +if ! kubectl config current-context | grep -q '^k3d-yucca$'; then + echo "Current kube context is not k3d-yucca. Run: mise k3d:up" >&2 + exit 1 +fi + +exec tilt ci --timeout 1800s "$@" diff --git a/.mise/tasks/tilt/ci-infra b/.mise/tasks/tilt/ci-infra new file mode 100755 index 00000000..eb53bf68 --- /dev/null +++ b/.mise/tasks/tilt/ci-infra @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +#MISE description="tilt ci for the infra subset only (no app images) — what integration tests need" +#MISE dir="{{config_root}}" +# The closure of every non-app resource: chart repos, operators, the Ceph dev +# cluster + RGW user, CNPG database, mock-oidc and the victoria pair. Skipping +# the four heavy app images (yucca-api/admin/web/michael) saves ~10 min in CI; +# integration tests boot those apps on the host themselves. +set -euo pipefail + +if ! kubectl config current-context | grep -q '^k3d-yucca$'; then + echo "Current kube context is not k3d-yucca. Run: mise k3d:up" >&2 + exit 1 +fi + +exec tilt ci --timeout 1800s \ + cloudnative-pg-repo rook-release-repo victoriametrics-repo helm-deps \ + cloudnative-pg rook-ceph-operator rook-ceph-cluster \ + yucca-object-user yucca-database yucca-mock-oidc \ + yucca-victoria-metrics yucca-victoria-logs diff --git a/.mise/tasks/tilt/down b/.mise/tasks/tilt/down new file mode 100755 index 00000000..a94722c0 --- /dev/null +++ b/.mise/tasks/tilt/down @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#MISE description="Tear down Tilt resources" +set -euo pipefail + +tilt down --delete-namespaces "$@" diff --git a/.mise/tasks/tilt/up b/.mise/tasks/tilt/up new file mode 100755 index 00000000..b6df3131 --- /dev/null +++ b/.mise/tasks/tilt/up @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +#MISE description="Start Tilt against the local k3d cluster" +set -euo pipefail + +if ! kubectl config current-context | grep -q '^k3d-yucca$'; then + echo "Current kube context is not k3d-yucca. Run: mise k3d:up" + exit 1 +fi + +tilt up "$@" diff --git a/.prettierignore b/.prettierignore index 6639213a..8299b9b0 100644 --- a/.prettierignore +++ b/.prettierignore @@ -31,5 +31,10 @@ yarn.lock ansible/ tf/ +# Helm charts: templates are Go-templated YAML (not parseable by prettier), +# and values/Chart files follow helm conventions. Validated by helm template +# + kubeconform via `mise k8s:validate` instead. +charts/ + # auto-generated yaak/ diff --git a/README.md b/README.md index 9103851f..ebda6fc9 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ If necessary, copy `.env.example` to `.env` and customise. Then use mise: ```bash -mise dev # install deps, prep environment, start servers +mise dev # install deps, prep environment, start servers (compose-based) mise check # lint, format check, svelte check @@ -28,13 +28,37 @@ mise test:e2e # e2e tests mise test:e2e:web # e2e web tests ``` +### Running on k3d + Tilt (Kubernetes) + +An alternative k8s-based dev flow mirrors the eventual prod topology (Helm charts, CloudNativePG, Rook-Ceph object storage, in-cluster service discovery). All required tools (k3d, kubectl, helm, tilt) are installed via mise. + +```bash +mise k3d:up # create local k3d cluster + registry +mise tilt:up # start Tilt; builds images, renders charts, port-forwards +# edit code — live_update syncs into running pods + +mise tilt:down # stop Tilt +mise k3d:down # delete cluster +``` + +Ports forwarded to localhost: + +- `5173` web, `3020` yucca-api, `3030` yucca-admin-api, `3010` michael +- `8092` mock-oidc, `9000` ceph rgw (S3) +- `8428` victoria-metrics, `9428` victoria-logs + +Tilt deploys the **same per-app charts the Flux tree uses** — it reads the +HelmReleases under [`kubernetes/apps`](./kubernetes) to discover what to deploy, +then builds/live-updates the images. Charts live in `charts/` (a `yucca-common` +library + per-service charts). See [`kubernetes/README.md`](./kubernetes/README.md). + ## Infrastructure -| Start here | Path | Purpose | -| ---------------------------------------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`ansible/ceph/README.md`](./ansible/ceph/README.md) | `ansible/ceph/` | Ansible automation for Ceph clusters (sietch, painbox). Deploys + operates via cephadm on bare-metal and Hetzner. | -| [`tf/README.md`](./tf/README.md) | `tf/` | Terraform/OpenTofu authority for cluster identity, 1P secret items, rendered Ansible inventories. Terragrunt multi-env (`deployment///`). | -| (coming in follow-up) | `kubernetes/` | Flux GitOps surface for the (future) Talos K8s cluster. | +| Start here | Path | Purpose | +| ---------------------------------------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`ansible/ceph/README.md`](./ansible/ceph/README.md) | `ansible/ceph/` | Ansible automation for Ceph clusters (sietch, painbox). Deploys + operates via cephadm on bare-metal and Hetzner. | +| [`tf/README.md`](./tf/README.md) | `tf/` | Terraform/OpenTofu authority for cluster identity, 1P secret items, rendered Ansible inventories. Terragrunt multi-env (`deployment///`). | +| [`kubernetes/README.md`](./kubernetes/README.md) | `kubernetes/` | Flux GitOps surface (apps/components/flux/bootstrap) for the (future) Talos K8s cluster. Per-app HelmReleases over the in-repo `charts/`; mirrored locally by Tilt. | **Secrets are managed via the `yucca_tf_*` 1Password vaults.** Runtime reads use a read-only service account; TF writes use a superuser service account. See diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 00000000..82e3a059 --- /dev/null +++ b/Tiltfile @@ -0,0 +1,377 @@ +# Yucca local dev on k3d + Tilt + Helm. +# +# Source of truth = the Flux tree under kubernetes/. Tilt derives EVERYTHING it +# deploys from there (see discover_apps below): +# - GitRepository-sourced HelmReleases -> the in-repo charts/, rendered +# with their dev defaults (charts/*/values.yaml) and live-updated with the +# locally-built images. +# - HelmRepository-sourced HelmReleases (cnpg, rook, victoria-*) -> installed +# at the exact chart version + values pinned in the HelmRelease, from the +# HelmRepositories declared in kubernetes/flux/repos/. +# APP_WIRING below carries only the dev-specific concerns Flux doesn't have: +# which locally-built image to inject, deploy ordering, and pod-readiness quirks. +# +# Service names are pinned via fullnameOverride in each chart, so in-cluster DNS +# is identical whether a chart is rendered here (release == resource name) or by +# Flux (per-app release names). + +load('ext://helm_resource', 'helm_resource', 'helm_repo') +load('ext://namespace', 'namespace_create') + +# Gate: only talk to the local k3d cluster. Prevents accidental prod deploys. +allow_k8s_contexts('k3d-yucca') + +namespace_create('yucca') + +# --------------------------------------------------------------------------- +# Optional dev secrets: a gitignored .env at the repo root (KEY=VALUE; values +# may be 1Password `op://` references, resolved here via `op read`). When +# present it becomes the yucca-dev-env Secret, layered onto yucca-api as the +# last envFrom source (last source wins for duplicate keys). read_file watches +# the path, so creating/editing .env retriggers automatically. Absent .env +# (CI, fresh clones) leaves the committed mock-oidc dev fixtures in charge. +# --------------------------------------------------------------------------- + +# Env vars the chart pins as explicit container env — explicit env always +# beats envFrom, so these .env keys must override through Helm values instead. +DEV_ENV_VALUE_KEYS = { + 'OIDC_ISSUER': 'oidcIssuer', + 'OIDC_REDIRECT_URI': 'oidcRedirectUri', + 'OIDC_LOGOUT_REDIRECT_URI': 'oidcLogoutRedirectUri', +} + +def load_dev_env(): + env = {} + for line in str(read_file('.env', default='')).splitlines(): + line = line.strip() + if line.startswith('export '): + line = line[len('export '):].lstrip() + if not line or line.startswith('#') or '=' not in line: + continue + key, _, value = line.partition('=') + env[key.strip()] = value.strip().strip('"').strip("'") + # OP_ACCOUNT picks the 1Password account on multi-account machines. It's + # loader config, not app env, so it never reaches the cluster. + account = env.pop('OP_ACCOUNT', '') + for key in env.keys(): + if env[key].startswith('op://'): + cmd = ['op', 'read', '--no-newline'] + (['--account', account] if account else []) + [env[key]] + # quiet/echo_off: keep resolved secrets out of the Tilt log. Fails + # loudly (aborting the Tiltfile) if op is missing or signed out. + env[key] = str(local(cmd, quiet=True, echo_off=True)) + return env + +DEV_ENV = load_dev_env() + +if DEV_ENV: + k8s_yaml(encode_yaml({ + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': {'name': 'yucca-dev-env', 'namespace': 'yucca'}, + 'stringData': DEV_ENV, + })) + k8s_resource(objects=['yucca-dev-env:secret'], new_name='dev-env', labels=['helm']) + +# --------------------------------------------------------------------------- +# Images. Built locally and injected into each app's Helm release via image_keys +# (image.repository/image.tag). Edits are live-synced into the running pods. +# --------------------------------------------------------------------------- +docker_build( + 'yucca-api', + context='.', + dockerfile='packages/yucca-api/Dockerfile', + target='dev', + # Rebuild only on package.json/lockfile/Dockerfile changes; src edits are + # handled by the syncs below (nest --watch picks them up in-pod). + only=[ + './pnpm-workspace.yaml', + './pnpm-lock.yaml', + './package.json', + './.npmrc', + './packages', + ], + ignore=[ + '**/node_modules', + '**/dist', + '**/.svelte-kit', + 'packages/michael', + 'packages/e2e', + ], + live_update=[ + sync('./packages/yucca-api', '/app/packages/yucca-api'), + sync('./packages/common', '/app/packages/common'), + run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']), + ], +) + +docker_build( + 'web', + context='.', + dockerfile='packages/web/Dockerfile', + target='dev', + only=[ + './pnpm-workspace.yaml', + './pnpm-lock.yaml', + './package.json', + './.npmrc', + './packages', + ], + ignore=[ + '**/node_modules', + '**/dist', + '**/.svelte-kit', + 'packages/michael', + 'packages/e2e', + ], + live_update=[ + sync('./packages/web', '/app/packages/web'), + sync('./packages/common', '/app/packages/common'), + sync('./packages/yucca-api-client', '/app/packages/yucca-api-client'), + sync('./packages/yucca-sdk', '/app/packages/yucca-sdk'), + run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']), + run('cd /app && pnpm --filter @futo-org/backups-api-client build', trigger=['./packages/yucca-api-client/src']), + run('cd /app && pnpm --filter @futo-org/backups-orchestrator-ui build', trigger=['./packages/yucca-sdk/orchestration-ui/src']), + run('cd /app && pnpm --filter web lingui:compile', trigger=['./packages/web/src/locales']), + ], +) + +docker_build( + 'michael', + context='.', + dockerfile='packages/michael/Dockerfile', + target='dev', + only=['./packages/michael'], + live_update=[ + sync('./packages/michael', '/src'), + run('cd /src && go mod download', trigger=['./packages/michael/go.sum']), + ], +) + +docker_build( + 'yucca-admin-api', + context='.', + dockerfile='packages/yucca-admin-api/Dockerfile', + target='dev', + only=[ + './pnpm-workspace.yaml', + './pnpm-lock.yaml', + './package.json', + './.npmrc', + './packages', + ], + ignore=[ + '**/node_modules', + '**/dist', + '**/.svelte-kit', + 'packages/michael', + 'packages/e2e', + ], + live_update=[ + sync('./packages/yucca-admin-api', '/app/packages/yucca-admin-api'), + sync('./packages/common', '/app/packages/common'), + # yucca-admin-api/src/schema is a symlink into yucca-api; keep it synced. + sync('./packages/yucca-api', '/app/packages/yucca-api'), + run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']), + ], +) + +# mock-oidc-provider has no dev target (config-only via env); a plain build is +# enough — it rarely changes and is reconfigured through Helm values. +docker_build( + 'mock-oidc-provider', + context='.', + dockerfile='packages/mock-oidc-provider/Dockerfile', + only=[ + './pnpm-workspace.yaml', + './pnpm-lock.yaml', + './package.json', + './.npmrc', + './packages/mock-oidc-provider', + ], + ignore=[ + '**/node_modules', + '**/dist', + ], +) + +# --------------------------------------------------------------------------- +# First-party Helm charts that depend on the yucca-common library need their +# subchart snapshot built before `helm upgrade` can render them. +# --------------------------------------------------------------------------- +local_resource( + 'helm-deps', + cmd='rm -rf charts/yucca-api/charts charts/yucca-admin-api/charts charts/web/charts charts/michael/charts charts/mock-oidc/charts && for d in charts/yucca-api charts/yucca-admin-api charts/web charts/michael charts/mock-oidc; do (cd $d && helm dependency build); done', + deps=[ + 'charts/yucca-api', + 'charts/yucca-admin-api', + 'charts/web', + 'charts/michael', + 'charts/mock-oidc', + 'charts/yucca-common', + ], + # `helm dependency build` rewrites these; if Tilt watches them we re-enter + # an infinite rebuild loop. + ignore=[ + 'charts/**/charts', + 'charts/**/charts/**', + 'charts/**/tmpcharts-*', + 'charts/**/tmpcharts-*/**', + 'charts/**/Chart.lock', + ], + labels=['helm'], +) + +# --------------------------------------------------------------------------- +# Deploy everything the Flux tree declares. The HelmRelease tree is the source +# of truth for WHAT runs (apps, operators, chart versions, remote values); the +# map below carries only the DEV concerns Flux doesn't know about. +# --------------------------------------------------------------------------- +APP_WIRING = { + # name (== HelmRelease metadata.name) build image ref resource_deps + # dev_env: receives the .env override Secret (see load_dev_env above). + 'yucca-api': {'build': 'yucca-api', 'deps': ['yucca-database', 'yucca-mock-oidc', 'yucca-michael'], 'dev_env': True}, + 'yucca-admin-api': {'build': 'yucca-admin-api', 'deps': ['yucca-database', 'yucca-mock-oidc']}, + 'yucca-web': {'build': 'web', 'deps': ['yucca-api']}, + 'yucca-michael': {'build': 'michael', 'deps': ['yucca-object-user']}, + 'yucca-mock-oidc': {'build': 'mock-oidc-provider', 'deps': []}, + 'yucca-database': {'build': None, 'deps': ['cloudnative-pg']}, + # The CephObjectStoreUser creates no pods (just a Secret once Rook mints the + # RGW user), so Tilt's pod tracking would hang at "pending". Mark ready on + # apply; michael still waits on this resource for ordering. + 'yucca-object-user': {'build': None, 'deps': ['rook-ceph-cluster'], 'pod_readiness': 'ignore'}, + # Rook spins up transient mon/osd "canary" pods and deletes them; Tilt's + # pod tracking misreads those deletions as failures. Ignore pod readiness + # here — real convergence is still gated downstream (object-user -> michael + # only go ready once the RGW + user secret actually exist). + 'rook-ceph-cluster': {'build': None, 'deps': ['rook-ceph-operator'], 'pod_readiness': 'ignore'}, + # Remote-chart operators/infra (HelmRepository-sourced). + 'cloudnative-pg': {'build': None, 'deps': []}, + 'rook-ceph-operator': {'build': None, 'deps': []}, + 'yucca-victoria-metrics': {'build': None, 'deps': []}, + 'yucca-victoria-logs': {'build': None, 'deps': []}, +} + +def discover_helm_repos(): + """HelmRepository name -> URL, from kubernetes/flux/repos/.""" + repos = {} + for path in listdir('kubernetes/flux/repos'): + if not path.endswith('.yaml'): + continue + doc = read_yaml(path) + if doc and doc.get('kind') == 'HelmRepository': + repos[doc['metadata']['name']] = doc['spec']['url'] + return repos + +def discover_apps(): + """All HelmReleases under kubernetes/apps, split by chart source kind.""" + local_apps, remote_apps = [], [] + for path in listdir('kubernetes/apps', recursive=True): + if not path.endswith('/helmrelease.yaml'): + continue + hr = read_yaml(path) + if not hr or hr.get('kind') != 'HelmRelease': + continue + chart_spec = hr['spec']['chart']['spec'] + source = chart_spec.get('sourceRef', {}) + chart = chart_spec.get('chart', '') + name = hr['metadata']['name'] + namespace = hr['metadata'].get('namespace', 'yucca') + if source.get('kind') == 'GitRepository' and chart.startswith('charts/'): + # In-repo chart: dev renders it with its values.yaml defaults; the + # HelmRelease's .spec.values are the prod-side overrides. + local_apps.append(struct(name=name, namespace=namespace, chart=chart)) + elif source.get('kind') == 'HelmRepository': + # Remote chart: dev installs the exact version + values Flux would. + remote_apps.append(struct( + name=name, + namespace=namespace, + chart=chart, + version=chart_spec.get('version', ''), + repo=source['name'], + values=hr['spec'].get('values', {}), + )) + return local_apps, remote_apps + +def wiring_for(app): + wiring = APP_WIRING.get(app.name) + if wiring == None: + fail("HelmRelease '%s' (%s) has no Tilt dev wiring — add it to APP_WIRING in the Tiltfile" % (app.name, app.chart)) + return wiring + +LOCAL_APPS, REMOTE_APPS = discover_apps() +HELM_REPOS = discover_helm_repos() + +for repo_name, url in HELM_REPOS.items(): + helm_repo('%s-repo' % repo_name, url, labels=['helm']) + +for app in REMOTE_APPS: + wiring = wiring_for(app) + flags = ['--create-namespace'] + if app.version: + flags += ['--version', app.version] + # Pass the HelmRelease's values verbatim (one --set-json per top-level key). + for key in sorted(app.values.keys()): + flags += ['--set-json', '%s=%s' % (key, str(encode_json(app.values[key])).rstrip('\n'))] + helm_resource( + app.name, + '%s-repo/%s' % (app.repo, app.chart), + namespace=app.namespace, + flags=flags, + resource_deps=['%s-repo' % app.repo] + wiring['deps'], + labels=['helm'], + pod_readiness=wiring.get('pod_readiness', ''), + ) + +for app in LOCAL_APPS: + wiring = wiring_for(app) + builds = [wiring['build']] if wiring['build'] else [] + flags = ['--timeout=10m'] + extra_deps = [] + if DEV_ENV and wiring.get('dev_env'): + flags += ['--set-json', 'extraEnvFrom=[{"secretRef":{"name":"yucca-dev-env"}}]'] + for key, value_path in DEV_ENV_VALUE_KEYS.items(): + if key in DEV_ENV: + flags += ['--set-string', '%s=%s' % (value_path, DEV_ENV[key])] + extra_deps = ['dev-env'] + helm_resource( + app.name, + app.chart, + namespace=app.namespace, + flags=flags, + image_deps=builds, + image_keys=[('image.repository', 'image.tag')] if builds else [], + resource_deps=['helm-deps'] + wiring['deps'] + extra_deps, + labels=['app'], + deps=[app.chart, 'charts/yucca-common'], + pod_readiness=wiring.get('pod_readiness', ''), + ) + +# --------------------------------------------------------------------------- +# Port-forwards. helm_resource bundles a release into one Tilt resource, so a +# single local_resource with raw kubectl tunnels gives each service its own. +# --------------------------------------------------------------------------- +local_resource( + 'port-forwards', + serve_cmd='''trap 'kill 0' EXIT +kubectl port-forward -n yucca svc/yucca-api 3020:3020 & +kubectl port-forward -n yucca svc/yucca-admin-api 3030:3030 & +kubectl port-forward -n yucca svc/yucca-web 5173:5173 & +kubectl port-forward -n yucca svc/yucca-michael 3010:3010 & +kubectl port-forward -n yucca svc/yucca-mock-oidc 8092:8092 & +kubectl port-forward -n rook-ceph svc/rook-ceph-rgw-yucca 9000:80 & +kubectl port-forward -n yucca svc/victoria-metrics 8428:8428 & +kubectl port-forward -n yucca svc/victoria-logs 9428:9428 & +wait''', + resource_deps=['yucca-api', 'yucca-web', 'yucca-michael', 'yucca-mock-oidc'], + labels=['app'], + links=[ + link('http://localhost:5173', 'web'), + link('http://localhost:3020', 'yucca-api'), + link('http://localhost:3030', 'yucca-admin-api'), + link('http://localhost:3010', 'michael'), + link('http://localhost:8092', 'mock-oidc'), + link('http://localhost:9000', 'ceph rgw (s3)'), + link('http://localhost:8428', 'victoria-metrics'), + link('http://localhost:9428', 'victoria-logs'), + ], +) diff --git a/charts/ceph-objectuser/Chart.yaml b/charts/ceph-objectuser/Chart.yaml new file mode 100644 index 00000000..02ac4700 --- /dev/null +++ b/charts/ceph-objectuser/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: ceph-objectuser +description: >- + CephObjectStoreUser for the dev Rook-Ceph object store. Creates a full RGW S3 + user (can create/manage buckets) and writes its credentials Secret into this + namespace. michael uses it (one bucket per restic repository). DEV ONLY. +type: application +version: 0.1.0 +appVersion: "0.0.1" diff --git a/charts/ceph-objectuser/templates/objectuser.yaml b/charts/ceph-objectuser/templates/objectuser.yaml new file mode 100644 index 00000000..f7355344 --- /dev/null +++ b/charts/ceph-objectuser/templates/objectuser.yaml @@ -0,0 +1,8 @@ +apiVersion: ceph.rook.io/v1 +kind: CephObjectStoreUser +metadata: + name: {{ .Values.userName }} +spec: + store: {{ .Values.store }} + clusterNamespace: {{ .Values.clusterNamespace }} + displayName: {{ .Values.userName }} diff --git a/charts/ceph-objectuser/values.yaml b/charts/ceph-objectuser/values.yaml new file mode 100644 index 00000000..ee90a76a --- /dev/null +++ b/charts/ceph-objectuser/values.yaml @@ -0,0 +1,7 @@ +# RGW user for michael. Rook writes a Secret named +# "rook-ceph-object-user--" into THIS namespace with keys +# AccessKey / SecretKey. michael (charts/michael) consumes them. +userName: michael +# CephObjectStore name + the namespace where the Rook cluster/objectstore lives. +store: yucca +clusterNamespace: rook-ceph diff --git a/charts/cnpg-cluster/Chart.yaml b/charts/cnpg-cluster/Chart.yaml new file mode 100644 index 00000000..22305e94 --- /dev/null +++ b/charts/cnpg-cluster/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cnpg-cluster +description: CloudNativePG Cluster for Yucca (consumes the cnpg operator's CRDs) +type: application +version: 0.1.0 +appVersion: "0.0.1" diff --git a/charts/cnpg-cluster/templates/cluster.yaml b/charts/cnpg-cluster/templates/cluster.yaml new file mode 100644 index 00000000..63edb119 --- /dev/null +++ b/charts/cnpg-cluster/templates/cluster.yaml @@ -0,0 +1,12 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ .Values.clusterName }} +spec: + instances: {{ .Values.instances }} + storage: + size: {{ .Values.storage }} + bootstrap: + initdb: + database: {{ .Values.database }} + owner: {{ .Values.owner }} diff --git a/charts/cnpg-cluster/values.yaml b/charts/cnpg-cluster/values.yaml new file mode 100644 index 00000000..b9f54e35 --- /dev/null +++ b/charts/cnpg-cluster/values.yaml @@ -0,0 +1,8 @@ +# CNPG Cluster. The name is the stable in-cluster identifier; yucca-api and +# yucca-admin-api derive their POSTGRES_* env from the "-app" secret +# that CNPG generates, so keep this consistent across dev and prod. +clusterName: yucca-db +instances: 1 +storage: 1Gi +database: yucca +owner: yucca diff --git a/charts/michael/Chart.yaml b/charts/michael/Chart.yaml new file mode 100644 index 00000000..d71991e7 --- /dev/null +++ b/charts/michael/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: michael +description: Yucca backup/restore service (Go) +type: application +version: 0.1.0 +appVersion: "0.0.1" +dependencies: + - name: yucca-common + version: 0.1.0 + repository: "file://../yucca-common" diff --git a/charts/michael/templates/deployment.yaml b/charts/michael/templates/deployment.yaml new file mode 100644 index 00000000..608d405f --- /dev/null +++ b/charts/michael/templates/deployment.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.deployment" . }} diff --git a/charts/michael/templates/secret.yaml b/charts/michael/templates/secret.yaml new file mode 100644 index 00000000..ab069f88 --- /dev/null +++ b/charts/michael/templates/secret.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.secret" . }} diff --git a/charts/michael/templates/service.yaml b/charts/michael/templates/service.yaml new file mode 100644 index 00000000..60a39083 --- /dev/null +++ b/charts/michael/templates/service.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.service" . }} diff --git a/charts/michael/values.yaml b/charts/michael/values.yaml new file mode 100644 index 00000000..622a7b5e --- /dev/null +++ b/charts/michael/values.yaml @@ -0,0 +1,76 @@ +replicas: 1 + +# Stable in-cluster name, independent of the Helm release name (dev == prod). +fullnameOverride: yucca-michael + +image: + repository: k3d-registry.localhost:5000/michael + tag: dev + pullPolicy: IfNotPresent + +ports: + - name: http + containerPort: 3010 + +service: + type: ClusterIP + +# DEV FIXTURE — the public half of the project's well-known local-dev keypair +# (see charts/yucca-api/values.yaml + .mise/tasks/*/env). Prod replaces this +# with an ExternalSecret. +secretData: + # michael (Go) verifies ES256 JWTs from yucca-api with this public key. + JWT_PUBLIC_KEY: | + -----BEGIN PUBLIC KEY----- + MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpLmwUSBO0p7P1UvGReVxFTvAsCfg + GS7NQtJ3AnJkYaigO1MS5J59I4uRXCmpLtcwTocUGHMRVZrGLQMWdZY4DQ== + -----END PUBLIC KEY----- + +env: + - name: RESTIC_API_PORT + value: "3010" + - name: LOG_LEVEL + value: debug + # S3 object store = Rook-Ceph RGW. michael creates one bucket per restic + # repository, so it needs a full RGW user (CephObjectStoreUser via + # charts/ceph-objectuser), NOT a bucket-scoped ObjectBucketClaim. Rook writes + # the user's keys into this namespace as rook-ceph-object-user--. + - name: S3_ENDPOINT + value: http://rook-ceph-rgw-yucca.rook-ceph.svc:80 + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: rook-ceph-object-user-yucca-michael + key: AccessKey + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: rook-ceph-object-user-yucca-michael + key: SecretKey + - name: S3_REGION + value: us-east-1 + - name: S3_FORCE_PATH_STYLE + value: "true" + - name: OTLP_METRICS_ENDPOINT + value: victoria-metrics:8428 + - name: OTLP_METRICS_URL_PATH + value: /opentelemetry/v1/metrics + - name: OTLP_LOGS_ENDPOINT + value: victoria-logs:9428 + - name: OTLP_LOGS_URL_PATH + value: /insert/opentelemetry/v1/logs + +envFrom: + - secretRef: + name: yucca-michael + +# Probes keep `tilt ci`/Flux honest: michael runs under air in dev, which keeps +# the container "Running" even when the binary fatals on boot. The TCP probe +# surfaces that as NotReady. No livenessProbe (air restarts are normal in dev). +startupProbe: + tcpSocket: { port: http } + periodSeconds: 2 + failureThreshold: 90 +readinessProbe: + tcpSocket: { port: http } + periodSeconds: 10 diff --git a/charts/mock-oidc/Chart.yaml b/charts/mock-oidc/Chart.yaml new file mode 100644 index 00000000..9416c2b1 --- /dev/null +++ b/charts/mock-oidc/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: mock-oidc +description: Mock OIDC provider for local dev (homegrown packages/mock-oidc-provider) +type: application +version: 0.1.0 +appVersion: "0.0.1" +dependencies: + - name: yucca-common + version: 0.1.0 + repository: "file://../yucca-common" diff --git a/charts/mock-oidc/templates/deployment.yaml b/charts/mock-oidc/templates/deployment.yaml new file mode 100644 index 00000000..608d405f --- /dev/null +++ b/charts/mock-oidc/templates/deployment.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.deployment" . }} diff --git a/charts/mock-oidc/templates/service.yaml b/charts/mock-oidc/templates/service.yaml new file mode 100644 index 00000000..60a39083 --- /dev/null +++ b/charts/mock-oidc/templates/service.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.service" . }} diff --git a/charts/mock-oidc/values.yaml b/charts/mock-oidc/values.yaml new file mode 100644 index 00000000..2bfca92d --- /dev/null +++ b/charts/mock-oidc/values.yaml @@ -0,0 +1,53 @@ +replicas: 1 + +# Stable in-cluster name, independent of the Helm release name (dev == prod). +# yucca-api/admin-api reference this as their OIDC issuer host. +fullnameOverride: yucca-mock-oidc + +image: + repository: k3d-registry.localhost:5000/mock-oidc-provider + tag: dev + pullPolicy: IfNotPresent + +# Container listens on 80 (see packages/mock-oidc-provider); exposed in-cluster +# and via port-forward as 8092 to match yucca-api's oidcIssuer. +ports: + - name: http + containerPort: 80 + servicePort: 8092 + +service: + type: ClusterIP + +env: + - name: PORT + value: "80" + # iss claim / discovery base — must match yucca-api's OIDC_ISSUER + # (http://yucca-mock-oidc:8092) so issued tokens validate in-cluster. + - name: ISSUER + value: http://yucca-mock-oidc:8092 + - name: CLIENT_ID + value: "client ID" + - name: CLIENT_SECRET + value: "client secret" + - name: DEVICE_CLIENT_ID + value: "device client ID" + # Browser-facing callbacks resolve via the Tilt port-forwards. + - name: REDIRECT_URI + value: http://localhost:5173/api/auth/oidc/callback + - name: POST_LOGOUT_REDIRECT_URI + value: http://localhost:5173 + - name: ADMIN_REDIRECT_URI + value: http://localhost:3030/api/auth/oidc/callback + - name: ADMIN_POST_LOGOUT_REDIRECT_URI + value: http://localhost:3030 + +# Probe the discovery document — it's what every consumer (yucca-api, the e2e, +# browsers) needs working. +startupProbe: + httpGet: { path: /.well-known/openid-configuration, port: http } + periodSeconds: 2 + failureThreshold: 60 +readinessProbe: + httpGet: { path: /.well-known/openid-configuration, port: http } + periodSeconds: 10 diff --git a/charts/rook-ceph-cluster/Chart.yaml b/charts/rook-ceph-cluster/Chart.yaml new file mode 100644 index 00000000..a1402452 --- /dev/null +++ b/charts/rook-ceph-cluster/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: rook-ceph-cluster +description: >- + Minimal single-node Rook-Ceph cluster + S3 object store for LOCAL DEV ONLY. + Replaces the dev MinIO object store. Not for production — prod storage is a + completely separate Ceph (see ansible/ceph + tf/). +type: application +version: 0.1.0 +appVersion: "1.20.0" diff --git a/charts/rook-ceph-cluster/templates/cephcluster.yaml b/charts/rook-ceph-cluster/templates/cephcluster.yaml new file mode 100644 index 00000000..cd99ae10 --- /dev/null +++ b/charts/rook-ceph-cluster/templates/cephcluster.yaml @@ -0,0 +1,46 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: {{ .Values.clusterName }} +spec: + dataDirHostPath: {{ .Values.dataDirHostPath }} + cephVersion: + image: {{ .Values.cephImage }} + # single-node, single-replica dev cluster is not a supported topology + allowUnsupported: true + mon: + count: {{ .Values.mon.count }} + allowMultiplePerNode: true + mgr: + count: {{ .Values.mgr.count }} + allowMultiplePerNode: true + dashboard: + enabled: {{ .Values.dashboard.enabled }} + # Trim the footprint for a laptop-sized dev cluster. + crashCollector: + disable: true + cephConfig: + global: + # no redundancy on a single OSD + osd_pool_default_size: "1" + osd_pool_default_min_size: "1" + mon_warn_on_pool_no_redundancy: "false" + mon_allow_pool_size_one: "true" + # Single-OSD dev fix: the PG autoscaler starts pools at pg_num=1, but Rook + # sets pg_num_min=8 on the RGW system pools (e.g. .rgw.root) -> EINVAL + # "pg_num_min 8 > pg_num 1". Disable autoscaling and default new pools to + # 8 PGs so the object store's pools are created at pg_num=8. + osd_pool_default_pg_autoscale_mode: "off" + osd_pool_default_pg_num: "8" + osd_pool_default_pgp_num: "8" + storage: + useAllNodes: true + useAllDevices: false + # Loop devices must be named explicitly (see values.yaml). Requires + # allowLoopDevices=true on the operator. + devices: + - name: {{ .Values.storage.device }} + healthCheck: + daemonHealth: + mon: + interval: 45s diff --git a/charts/rook-ceph-cluster/templates/cephobjectstore.yaml b/charts/rook-ceph-cluster/templates/cephobjectstore.yaml new file mode 100644 index 00000000..41324fa4 --- /dev/null +++ b/charts/rook-ceph-cluster/templates/cephobjectstore.yaml @@ -0,0 +1,25 @@ +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: {{ .Values.objectStore.name }} +spec: + metadataPool: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false + dataPool: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false + preservePoolsOnDelete: false + gateway: + port: {{ .Values.objectStore.gateway.port }} + instances: {{ .Values.objectStore.gateway.instances }} + # Allow CephObjectStoreUser CRs in the app namespace (so the user's S3 secret + # is written there for michael to consume). See charts/ceph-objectuser. + allowUsersInNamespaces: + {{- range .Values.objectStore.allowUsersInNamespaces }} + - {{ . }} + {{- end }} diff --git a/charts/rook-ceph-cluster/templates/loop-device.yaml b/charts/rook-ceph-cluster/templates/loop-device.yaml new file mode 100644 index 00000000..83f4e68d --- /dev/null +++ b/charts/rook-ceph-cluster/templates/loop-device.yaml @@ -0,0 +1,73 @@ +{{- if .Values.loopDevice.enabled }} +# DEV ONLY. k3d nodes have no spare block device, and Rook OSDs need raw block +# storage (k3d's local-path provisioner is filesystem-only). This privileged +# DaemonSet creates a sparse image on the node and losetup-attaches it to a +# FIXED loop device (storage.device / loopDevice.device) that the CephCluster +# names explicitly. The container stays alive so the attachment persists. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Values.clusterName }}-loop-device + labels: + app.kubernetes.io/name: rook-ceph-loop-device +spec: + selector: + matchLabels: + app.kubernetes.io/name: rook-ceph-loop-device + template: + metadata: + labels: + app.kubernetes.io/name: rook-ceph-loop-device + spec: + hostPID: true + containers: + - name: loop-device + image: {{ .Values.loopDevice.image }} + securityContext: + privileged: true + command: ["/bin/sh", "-c"] + args: + - | + set -eu + apk add --no-cache util-linux >/dev/null 2>&1 || true + IMG="{{ .Values.loopDevice.path }}" + DEV="{{ .Values.loopDevice.device }}" + mkdir -p "$(dirname "$IMG")" + [ -f "$IMG" ] || truncate -s {{ .Values.loopDevice.sizeGiB }}G "$IMG" + # Attach the image to the FIXED device the CephCluster references. + # Loop attachments live in the HOST kernel (shared by every k3d + # node), so they survive cluster re-creation: after a k3d:reset, + # DEV is typically still bound to the PREVIOUS cluster's now- + # deleted image — which also makes Rook skip it (stale bluestore + # signature). Clean all stale state before attaching: + if losetup "$DEV" >/dev/null 2>&1; then + back="$(losetup --noheadings --output BACK-FILE "$DEV" 2>/dev/null | xargs || true)" + # detach unless it's a live attachment of exactly our image + [ "$back" = "$IMG" ] || losetup -d "$DEV" || true + fi + # our image attached on some other device = corruption hazard + for d in $(losetup -j "$IMG" | cut -d: -f1); do + [ "$d" = "$DEV" ] || losetup -d "$d" || true + done + if ! losetup "$DEV" >/dev/null 2>&1; then + [ -e "$DEV" ] || mknod "$DEV" b 7 "${DEV#/dev/loop}" + losetup "$DEV" "$IMG" + fi + echo "loop device for Rook OSD: $(losetup -j "$IMG" | cut -d: -f1) ($IMG)" + # keep the container (and thus the loop attachment) alive + while true; do sleep 3600; done + volumeMounts: + - name: dev + mountPath: /dev + mountPropagation: Bidirectional + - name: rook-data + mountPath: {{ dir .Values.loopDevice.path }} + volumes: + - name: dev + hostPath: + path: /dev + - name: rook-data + hostPath: + path: {{ dir .Values.loopDevice.path }} + type: DirectoryOrCreate +{{- end }} diff --git a/charts/rook-ceph-cluster/values.yaml b/charts/rook-ceph-cluster/values.yaml new file mode 100644 index 00000000..8eb2b97d --- /dev/null +++ b/charts/rook-ceph-cluster/values.yaml @@ -0,0 +1,44 @@ +# Minimal single-node Rook-Ceph for local dev (k3d). DEV ONLY. +clusterName: rook-ceph +dataDirHostPath: /var/lib/rook +# Pin to match the rook-ceph operator appVersion (Renovate keeps these in step). +cephImage: quay.io/ceph/ceph:v19.2.2 + +mon: + count: 1 +mgr: + count: 1 +dashboard: + enabled: false + +storage: + # Rook does NOT auto-select loop devices via useAllDevices (even with + # allowLoopDevices on the operator) — they must be named explicitly. This is + # the loop device attached by the DaemonSet below. A HIGH minor number: the + # kernel hands out the lowest free loop devices, so k3s/containerd will have + # claimed loop0..N on a fresh node — loop100 is reliably ours. + device: /dev/loop100 + +# S3 object store (CephObjectStore + RGW). +objectStore: + name: yucca + region: us-east-1 + gateway: + port: 80 + instances: 1 + # Namespaces allowed to host CephObjectStoreUser CRs (michael's S3 user lives + # in the yucca namespace so its secret is written there). + allowUsersInNamespaces: + - yucca + +# k3d has no spare block device, so synthesize one with a loopback file. This +# privileged DaemonSet truncates a sparse image on the node and losetup-attaches +# it; Rook's OSD then consumes it. Disable if your nodes have real disks. +loopDevice: + enabled: true + image: alpine:3.21 + sizeGiB: 20 + # where the backing image file lives on the node (under dataDirHostPath) + path: /var/lib/rook/osd-loop.img + # fixed device the image is attached to (must match storage.device above) + device: /dev/loop100 diff --git a/charts/web/Chart.yaml b/charts/web/Chart.yaml new file mode 100644 index 00000000..9bceed39 --- /dev/null +++ b/charts/web/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: web +description: Yucca web UI (SvelteKit SSR) +type: application +version: 0.1.0 +appVersion: "0.0.1" +dependencies: + - name: yucca-common + version: 0.1.0 + repository: "file://../yucca-common" diff --git a/charts/web/templates/deployment.yaml b/charts/web/templates/deployment.yaml new file mode 100644 index 00000000..608d405f --- /dev/null +++ b/charts/web/templates/deployment.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.deployment" . }} diff --git a/charts/web/templates/service.yaml b/charts/web/templates/service.yaml new file mode 100644 index 00000000..60a39083 --- /dev/null +++ b/charts/web/templates/service.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.service" . }} diff --git a/charts/web/values.yaml b/charts/web/values.yaml new file mode 100644 index 00000000..a960056d --- /dev/null +++ b/charts/web/values.yaml @@ -0,0 +1,38 @@ +replicas: 1 + +# Stable in-cluster name, independent of the Helm release name (dev == prod). +fullnameOverride: yucca-web + +image: + repository: k3d-registry.localhost:5000/web + tag: dev + pullPolicy: IfNotPresent + +ports: + - name: http + containerPort: 5173 + +service: + type: ClusterIP + +env: + - name: NODE_ENV + value: development + - name: PUBLIC_API_URL + value: http://yucca-api:3020 + - name: YUCCA_API_URL + value: http://yucca-api:3020/ + +# Probes keep `tilt ci`/Flux honest: vite binds the port even when SSR is +# broken (a stale workspace lib once made every page a 500 while the pod looked +# Ready), so probe with a real GET /. Generous timeouts — the first request +# triggers vite's initial compile. +startupProbe: + httpGet: { path: /, port: http } + periodSeconds: 5 + timeoutSeconds: 10 + failureThreshold: 60 +readinessProbe: + httpGet: { path: /, port: http } + periodSeconds: 15 + timeoutSeconds: 10 diff --git a/charts/yucca-admin-api/Chart.yaml b/charts/yucca-admin-api/Chart.yaml new file mode 100644 index 00000000..c5365fcd --- /dev/null +++ b/charts/yucca-admin-api/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: yucca-admin-api +description: Yucca admin API (NestJS) +type: application +version: 0.1.0 +appVersion: "0.0.1" +dependencies: + - name: yucca-common + version: 0.1.0 + repository: "file://../yucca-common" diff --git a/charts/yucca-admin-api/templates/deployment.yaml b/charts/yucca-admin-api/templates/deployment.yaml new file mode 100644 index 00000000..f882dd42 --- /dev/null +++ b/charts/yucca-admin-api/templates/deployment.yaml @@ -0,0 +1,12 @@ +{{- $_ := set .Values "env" (concat .Values.env (list + (dict "name" "OIDC_ADMIN_ISSUER" "value" .Values.oidcIssuer) + (dict "name" "OIDC_ADMIN_REDIRECT_URI" "value" .Values.oidcRedirectUri) + (dict "name" "OIDC_ADMIN_LOGOUT_REDIRECT_URI" "value" .Values.oidcLogoutRedirectUri) + (dict "name" "POSTGRES_HOST" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "host"))) + (dict "name" "POSTGRES_PORT" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "port"))) + (dict "name" "POSTGRES_DATABASE" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "dbname"))) + (dict "name" "POSTGRES_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "username"))) + (dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password"))) +)) }} +{{- $_ := set .Values "envFrom" (list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .)))) }} +{{- include "yucca-common.deployment" . }} diff --git a/charts/yucca-admin-api/templates/secret.yaml b/charts/yucca-admin-api/templates/secret.yaml new file mode 100644 index 00000000..ab069f88 --- /dev/null +++ b/charts/yucca-admin-api/templates/secret.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.secret" . }} diff --git a/charts/yucca-admin-api/templates/service.yaml b/charts/yucca-admin-api/templates/service.yaml new file mode 100644 index 00000000..60a39083 --- /dev/null +++ b/charts/yucca-admin-api/templates/service.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.service" . }} diff --git a/charts/yucca-admin-api/values.yaml b/charts/yucca-admin-api/values.yaml new file mode 100644 index 00000000..073782cd --- /dev/null +++ b/charts/yucca-admin-api/values.yaml @@ -0,0 +1,58 @@ +replicas: 1 + +# Stable in-cluster name, independent of the Helm release name (dev == prod). +fullnameOverride: yucca-admin-api + +image: + repository: k3d-registry.localhost:5000/yucca-admin-api + tag: dev + pullPolicy: IfNotPresent + +ports: + - name: http + containerPort: 3030 + +service: + type: ClusterIP + +# CNPG-managed postgres Cluster name (shares yucca-api's database) +postgresClusterName: yucca-db + +# OIDC provider in-cluster service (must match the issuer mock-oidc advertises) +oidcIssuer: http://yucca-mock-oidc:8092 +oidcRedirectUri: http://localhost:3030/api/auth/oidc/callback +oidcLogoutRedirectUri: http://localhost:3030 + +# Static dev secrets (overridden in prod via ExternalSecret) +secretData: + OIDC_ADMIN_CLIENT_ID: "client ID" + OIDC_ADMIN_CLIENT_SECRET: "client secret" + +env: + - name: NODE_ENV + value: development + - name: YUCCA_ADMIN_API_PORT + value: "3030" + - name: LOG_LEVEL + value: debug + - name: OIDC_ADMIN_ALLOW_INSECURE + value: "true" + - name: OTLP_METRICS_ENDPOINT + value: victoria-metrics:8428 + - name: OTLP_METRICS_URL_PATH + value: /opentelemetry/v1/metrics + - name: OTLP_LOGS_ENDPOINT + value: victoria-logs:9428 + - name: OTLP_LOGS_URL_PATH + value: /insert/opentelemetry/v1/logs + +# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process +# still counts as Ready (this masked two real bugs). The startupProbe budgets +# for the dev watcher's first boot; no livenessProbe so dev never restart-loops. +startupProbe: + tcpSocket: { port: http } + periodSeconds: 5 + failureThreshold: 60 +readinessProbe: + tcpSocket: { port: http } + periodSeconds: 10 diff --git a/charts/yucca-api/Chart.yaml b/charts/yucca-api/Chart.yaml new file mode 100644 index 00000000..091b0d94 --- /dev/null +++ b/charts/yucca-api/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: yucca-api +description: Yucca REST API (NestJS) +type: application +version: 0.1.0 +appVersion: "0.0.1" +dependencies: + - name: yucca-common + version: 0.1.0 + repository: "file://../yucca-common" diff --git a/charts/yucca-api/templates/deployment.yaml b/charts/yucca-api/templates/deployment.yaml new file mode 100644 index 00000000..1ea433be --- /dev/null +++ b/charts/yucca-api/templates/deployment.yaml @@ -0,0 +1,16 @@ +{{- $_ := set .Values "env" (concat .Values.env (list + (dict "name" "OIDC_ISSUER" "value" .Values.oidcIssuer) + (dict "name" "OIDC_REDIRECT_URI" "value" .Values.oidcRedirectUri) + (dict "name" "OIDC_LOGOUT_REDIRECT_URI" "value" .Values.oidcLogoutRedirectUri) + (dict "name" "POSTGRES_HOST" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "host"))) + (dict "name" "POSTGRES_PORT" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "port"))) + (dict "name" "POSTGRES_DATABASE" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "dbname"))) + (dict "name" "POSTGRES_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "username"))) + (dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password"))) +)) }} +{{- /* extraEnvFrom comes after the chart secret: for duplicate keys Kubernetes +takes the LAST envFrom source, so these act as overrides. */}} +{{- $_ := set .Values "envFrom" (concat + (list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .)))) + (.Values.extraEnvFrom | default (list))) }} +{{- include "yucca-common.deployment" . }} diff --git a/charts/yucca-api/templates/migration-job.yaml b/charts/yucca-api/templates/migration-job.yaml new file mode 100644 index 00000000..82cca009 --- /dev/null +++ b/charts/yucca-api/templates/migration-job.yaml @@ -0,0 +1,51 @@ +{{- if .Values.migration.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "yucca-common.fullname" . }}-migrate + labels: + {{- include "yucca-common.labels" . | nindent 4 }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 10 + template: + metadata: + labels: + {{- include "yucca-common.selectorLabels" . | nindent 8 }} + job: migrate + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-pg + image: postgres:18-alpine + command: ["sh", "-c"] + args: + - | + until pg_isready -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USERNAME"; do + echo "waiting for postgres..."; sleep 2; + done + env: + - { name: POSTGRES_HOST, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: host } } } + - { name: POSTGRES_PORT, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: port } } } + - { name: POSTGRES_USERNAME, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: username } } } + containers: + - name: migrate + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }} + workingDir: /app + command: ["sh", "-c"] + args: + - | + pnpm install --frozen-lockfile + cd packages/yucca-api + pnpm exec sql-tools -u "postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}" migrate + env: + - { name: POSTGRES_HOST, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: host } } } + - { name: POSTGRES_PORT, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: port } } } + - { name: POSTGRES_DATABASE, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: dbname } } } + - { name: POSTGRES_USERNAME, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: username } } } + - { name: POSTGRES_PASSWORD, valueFrom: { secretKeyRef: { name: {{ printf "%s-app" .Values.postgresClusterName }}, key: password } } } +{{- end }} diff --git a/charts/yucca-api/templates/secret.yaml b/charts/yucca-api/templates/secret.yaml new file mode 100644 index 00000000..ab069f88 --- /dev/null +++ b/charts/yucca-api/templates/secret.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.secret" . }} diff --git a/charts/yucca-api/templates/service.yaml b/charts/yucca-api/templates/service.yaml new file mode 100644 index 00000000..60a39083 --- /dev/null +++ b/charts/yucca-api/templates/service.yaml @@ -0,0 +1 @@ +{{- include "yucca-common.service" . }} diff --git a/charts/yucca-api/values.yaml b/charts/yucca-api/values.yaml new file mode 100644 index 00000000..03cdce5c --- /dev/null +++ b/charts/yucca-api/values.yaml @@ -0,0 +1,93 @@ +replicas: 1 + +# Stable in-cluster name, independent of the Helm release name. This keeps +# service DNS identical whether rendered by Tilt (dev) or Flux (prod, per-app +# release names). +fullnameOverride: yucca-api + +image: + repository: k3d-registry.localhost:5000/yucca-api + tag: dev + pullPolicy: IfNotPresent + +ports: + - name: http + containerPort: 3020 + +service: + type: ClusterIP + +# CNPG-managed postgres Cluster name (see umbrella chart) +postgresClusterName: yucca-db + +# OIDC provider in-cluster service +oidcIssuer: http://yucca-mock-oidc:8092 +oidcRedirectUri: http://localhost:5173/api/auth/oidc/callback +oidcLogoutRedirectUri: http://localhost:5173 + +# DEV FIXTURES — not secrets. This is the project's well-known local-dev +# keypair (the same one committed in .mise/tasks/*/env); yucca-api signs +# device/restic JWTs with it and michael verifies with the matching public key. +# It must never protect anything real. Prod replaces this whole block with +# ExternalSecrets (1Password) — see kubernetes/README.md. +secretData: + JWT_PRIVATE_KEY: | + -----BEGIN PRIVATE KEY----- + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgCla79+Sip4o2hZ1K + nQJYp2xU/nsCJmRoZmvXjeX6cW+hRANCAASkubBRIE7Sns/VS8ZF5XEVO8CwJ+AZ + Ls1C0ncCcmRhqKA7UxLknn0ji5FcKaku1zBOhxQYcxFVmsYtAxZ1ljgN + -----END PRIVATE KEY----- + OIDC_CLIENT_ID: "client ID" + OIDC_CLIENT_SECRET: "client secret" + +# Extra envFrom sources appended AFTER the chart's own secret — for duplicate +# keys the last source wins, so this is the override hook. Tilt points it at +# the yucca-dev-env Secret (built from a gitignored root .env, op:// refs +# resolved via the 1Password CLI); prod can point it at an ExternalSecret. +# NB: explicit `env` entries below always beat envFrom — env vars the chart +# pins there (OIDC_ISSUER & co) are overridden via their Helm values instead. +extraEnvFrom: [] + +env: + - name: NODE_ENV + value: development + - name: YUCCA_API_PORT + value: "3020" + - name: LOG_LEVEL + value: debug + - name: OIDC_ALLOW_INSECURE + value: "true" + # Device-flow OIDC (required by yucca-api env schema; points at mock-oidc's + # registered device client). + - name: OIDC_DEVICE_ISSUER + value: http://yucca-mock-oidc:8092 + - name: OIDC_DEVICE_CLIENT_ID + value: "device client ID" + - name: OIDC_DEVICE_ALLOW_INSECURE + value: "true" + - name: RESTIC_API_HOST + value: yucca-michael + - name: RESTIC_API_PORT + value: "3010" + - name: OTLP_METRICS_ENDPOINT + value: victoria-metrics:8428 + - name: OTLP_METRICS_URL_PATH + value: /opentelemetry/v1/metrics + - name: OTLP_LOGS_ENDPOINT + value: victoria-logs:9428 + - name: OTLP_LOGS_URL_PATH + value: /insert/opentelemetry/v1/logs + +# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process +# still counts as Ready (this masked two real bugs). The startupProbe budgets +# for the dev watcher's first boot; no livenessProbe so dev never restart-loops. +startupProbe: + tcpSocket: { port: http } + periodSeconds: 5 + failureThreshold: 60 +readinessProbe: + tcpSocket: { port: http } + periodSeconds: 10 + +migration: + enabled: false diff --git a/charts/yucca-common/Chart.yaml b/charts/yucca-common/Chart.yaml new file mode 100644 index 00000000..a665fa05 --- /dev/null +++ b/charts/yucca-common/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: yucca-common +description: Library chart with shared templates for Yucca services +type: library +version: 0.1.0 diff --git a/charts/yucca-common/templates/_deployment.tpl b/charts/yucca-common/templates/_deployment.tpl new file mode 100644 index 00000000..c6a5ec1a --- /dev/null +++ b/charts/yucca-common/templates/_deployment.tpl @@ -0,0 +1,70 @@ +{{- define "yucca-common.deployment" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "yucca-common.fullname" . }} + labels: + {{- include "yucca-common.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicas | default 1 }} + selector: + matchLabels: + {{- include "yucca-common.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "yucca-common.selectorLabels" . | nindent 8 }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.serviceAccountName }} + serviceAccountName: {{ . }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }} + {{- with .Values.command }} + command: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.args }} + args: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.workingDir }} + workingDir: {{ . }} + {{- end }} + ports: + {{- range .Values.ports }} + - name: {{ .name }} + containerPort: {{ .containerPort }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.startupProbe }} + startupProbe: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/charts/yucca-common/templates/_helpers.tpl b/charts/yucca-common/templates/_helpers.tpl new file mode 100644 index 00000000..a245963d --- /dev/null +++ b/charts/yucca-common/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{/* +Fully qualified app name. Falls back to .Release.Name-.Chart.Name. +*/}} +{{- define "yucca-common.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "yucca-common.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "yucca-common.labels" -}} +app.kubernetes.io/name: {{ include "yucca-common.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end -}} + +{{- define "yucca-common.selectorLabels" -}} +app.kubernetes.io/name: {{ include "yucca-common.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/yucca-common/templates/_secret.tpl b/charts/yucca-common/templates/_secret.tpl new file mode 100644 index 00000000..729a155f --- /dev/null +++ b/charts/yucca-common/templates/_secret.tpl @@ -0,0 +1,13 @@ +{{- define "yucca-common.secret" -}} +{{- if .Values.secretData -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "yucca-common.fullname" . }} + labels: + {{- include "yucca-common.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- toYaml .Values.secretData | nindent 2 }} +{{- end -}} +{{- end -}} diff --git a/charts/yucca-common/templates/_service.tpl b/charts/yucca-common/templates/_service.tpl new file mode 100644 index 00000000..bc4d8e1e --- /dev/null +++ b/charts/yucca-common/templates/_service.tpl @@ -0,0 +1,19 @@ +{{- define "yucca-common.service" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "yucca-common.fullname" . }} + labels: + {{- include "yucca-common.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type | default "ClusterIP" }} + selector: + {{- include "yucca-common.selectorLabels" . | nindent 4 }} + ports: + {{- range .Values.ports }} + - name: {{ .name }} + port: {{ .servicePort | default .containerPort }} + targetPort: {{ .name }} + protocol: {{ .protocol | default "TCP" }} + {{- end }} +{{- end -}} diff --git a/k3d.yaml b/k3d.yaml new file mode 100644 index 00000000..de765327 --- /dev/null +++ b/k3d.yaml @@ -0,0 +1,27 @@ +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: yucca +servers: 1 +agents: 0 +image: rancher/k3s:v1.32.2-k3s1 +registries: + create: + name: k3d-registry.localhost + host: '0.0.0.0' + hostPort: '5000' +options: + k3d: + wait: true + timeout: '60s' + k3s: + extraArgs: + - arg: '--disable=traefik' + nodeFilters: ['server:*'] + - arg: '--disable=servicelb' + nodeFilters: ['server:*'] + - arg: '--disable=metrics-server' + nodeFilters: ['server:*'] + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 00000000..cb0fc3b7 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,140 @@ +# Kubernetes (Flux GitOps) + +Flux GitOps surface for the Yucca cluster, laid out in the +[home-operations](https://github.com/onedr0p/cluster-template) convention: + +``` +kubernetes/ +├── bootstrap/ # one-time Flux install notes for a fresh cluster +├── flux/ # Flux sources (GitRepository/HelmRepository) + cluster entrypoint +├── components/ # reusable Kustomize components (cross-app concerns) +└── apps/ # applications, grouped by namespace + ├── cnpg-system/ # CloudNativePG operator + ├── rook-ceph/ # Rook-Ceph operator + dev cluster (S3 object store) + └── yucca/ # the product stack + its dev infra +``` + +## End-to-end tests against the local k3d stack + +The e2e suites (`packages/e2e` + the web Playwright test) are written for a +host-local environment, while this stack runs in k3d. One command bridges them: + +```bash +mise k3d:up && mise tilt:up # stack healthy (context k3d-yucca) +mise test:e2e:k3d # runs all e2e against the cluster +``` + +`mise test:e2e:k3d` (see `packages/e2e/k3d/run.sh`): + +- **orchestration-api runs as a separate local process** (`:22676`) — it is + intentionally _not_ deployed to k8s; it targets the port-forwarded web. +- port-forwards the cluster services to the host ports the suites expect + (michael `:3010`, yucca-api `:3020`, mock-oidc `:8092`, web `:36033` + `:5173`). +- resolves the in-cluster OIDC issuer host (`yucca-mock-oidc`) on the host with + a Node DNS preload (jest) and Chromium `--host-resolver-rules` (Playwright) — + no `/etc/hosts`/sudo needed. +- sets `RESTIC_ENDPOINT=localhost:3010` on yucca-api **for the test run only** + (restic runs on the host; the chart keeps the in-cluster `yucca-michael`), + reverted on exit. + +Note: michael creates **one S3 bucket per restic repository** (the bucket name +comes from the client JWT's `repository` claim; the S3 credentials are a static +RGW user). That's why it uses a full `CephObjectStoreUser` (charts/ceph-objectuser) +rather than a bucket-scoped ObjectBucketClaim. + +## How it reconciles + +Flux applies `kubernetes/flux/cluster` first (`cluster-repos` → `cluster-apps`). +`cluster-apps` builds `kubernetes/apps`, which aggregates one Flux `Kustomization` +(`ks.yaml`) per app. Each `ks.yaml` reconciles its `app/` directory, whose +`kustomization.yaml` applies a single `HelmRelease`. Ordering is expressed with +`dependsOn` (operator → database → apps). + +``` +apps/yucca// +├── ks.yaml # Flux Kustomization → ./app (+ dependsOn) +└── app/ + ├── kustomization.yaml + └── helmrelease.yaml # chart ref + values +``` + +## Single source of truth: this tree + +The [Tiltfile](../Tiltfile) derives **everything it deploys** from the +HelmReleases here — first-party apps _and_ the remote-chart operators: + +- First-party `HelmRelease`s reference the in-repo Helm charts via the `yucca` + `GitRepository` source (`chart: charts/`), so **no OCI publishing is + required**. Tilt renders the same charts with their dev defaults and injects + the locally-built, live-updated images. +- Remote `HelmRelease`s (cnpg, rook, victoria-\*) pin a chart version + values; + Tilt installs **exactly those**, from the `HelmRepository` sources declared in + `flux/repos/`. Bump a version or value once, in the HelmRelease — there is no + second copy to drift. + +Service names are pinned with `fullnameOverride` in each chart's `values.yaml`, +so in-cluster DNS is identical whether a chart is rendered by Tilt (release +`yucca`) or by Flux (per-app release names). + +| Layer | Reconciler | Image source | First-party values | +| -------- | ---------- | ------------------------------------------- | ------------------------------------------ | +| **Dev** | Tilt | `docker_build` → k3d registry, live-updated | chart defaults (`values.yaml`) | +| **Prod** | Flux | `ghcr.io/...` (per `HelmRelease`) | chart defaults + `HelmRelease.spec.values` | + +## Dev vs prod + +This tree currently mirrors the **dev** stack so it stays 1:1 with Tilt. Items +marked `TODO(prod)` (image registries, real OIDC/S3 endpoints, ingress, probes, +persistence, secrets) are where a future prod cluster overlay diverges. Notably: + +- `mock-oidc` and `rook-ceph` are **dev-only**. Prod swaps in a real IdP, and + prod object storage is a **completely separate** Ceph (the bare-metal cluster + in [`ansible/ceph`](../ansible/ceph) / [`tf/`](../tf)) — not this Rook cluster. +- The Rook-Ceph dev cluster is single-node/single-replica and synthesizes a + loopback block device (k3d has no spare disk). See + [`charts/rook-ceph-cluster`](../charts/rook-ceph-cluster). `michael`'s S3 + credentials come from a full RGW user + ([`charts/ceph-objectuser`](../charts/ceph-objectuser)) whose Secret Rook + writes into the `yucca` namespace. +- The dev keypair/secrets committed in chart `secretData` are **well-known + fixtures** (the same keypair lives in `.mise/tasks/*/env`); they must become + `ExternalSecret`s backed by the org's 1Password (External Secrets Operator) + before prod. + +## Real OIDC credentials in dev (`.env` + 1Password) + +The k3d stack runs against mock-oidc out of the box. To point `yucca-api` at a +real IdP, drop a (gitignored) `.env` at the repo root — values may be +1Password `op://` references, resolved through the `op` CLI when the Tiltfile +loads: + +```bash +OP_ACCOUNT="team-futo.1password.com" # only needed with multiple 1P accounts +OIDC_ISSUER="https://external-dev-gkhk8b.us1.zitadel.cloud" +OIDC_CLIENT_ID="op://yucca_tf_dev/CUSTOMER_ZITADEL_OAUTH_CLIENT_ID_DEV_TEST/password" +OIDC_CLIENT_SECRET="op://yucca_tf_dev/CUSTOMER_ZITADEL_OAUTH_CLIENT_SECRET_DEV_TEST/password" +``` + +Tilt turns the resolved pairs into the `yucca-dev-env` Secret and layers it +onto `yucca-api` as its last `envFrom` source (last source wins), so any key +here overrides the committed dev fixtures. `OIDC_ISSUER`/`OIDC_REDIRECT_URI`/ +`OIDC_LOGOUT_REDIRECT_URI` are pinned by the chart as explicit env (which +beats `envFrom`) and are mapped onto their Helm values instead — keep those +three non-secret, as Helm flags are visible in the Tilt UI. Editing or +deleting `.env` redeploys automatically; without it (CI, fresh clones) +nothing changes. + +Caveats: the IdP must allow `http://localhost:5173/api/auth/oidc/callback` as +a redirect URI; the device flow (`OIDC_DEVICE_*`) stays on mock-oidc; and the +web e2e suite logs in via mock-oidc, so remove `.env` before +`mise test:e2e:k3d`. + +## Validate locally + +```bash +# render the kustomize graph +kubectl kustomize kubernetes/apps +# build the whole tree (Kustomizations + HelmReleases) the way Flux would +# (https://github.com/allenporter/flux-local) +flux-local build all kubernetes --enable-helm --no-enable-dns +``` diff --git a/kubernetes/apps/cnpg-system/cloudnative-pg/app/helmrelease.yaml b/kubernetes/apps/cnpg-system/cloudnative-pg/app/helmrelease.yaml new file mode 100644 index 00000000..1dfead91 --- /dev/null +++ b/kubernetes/apps/cnpg-system/cloudnative-pg/app/helmrelease.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cloudnative-pg + namespace: cnpg-system +spec: + interval: 1h + chart: + spec: + chart: cloudnative-pg + version: 0.23.0 + sourceRef: + kind: HelmRepository + name: cloudnative-pg + namespace: flux-system + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + crds: CreateReplace + remediation: + retries: 3 diff --git a/kubernetes/apps/cnpg-system/cloudnative-pg/app/kustomization.yaml b/kubernetes/apps/cnpg-system/cloudnative-pg/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/cnpg-system/cloudnative-pg/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/cnpg-system/cloudnative-pg/ks.yaml b/kubernetes/apps/cnpg-system/cloudnative-pg/ks.yaml new file mode 100644 index 00000000..c05653e6 --- /dev/null +++ b/kubernetes/apps/cnpg-system/cloudnative-pg/ks.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cnpg-operator + namespace: flux-system +spec: + targetNamespace: cnpg-system + path: ./kubernetes/apps/cnpg-system/cloudnative-pg/app + prune: true + wait: true + interval: 1h + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/apps/cnpg-system/kustomization.yaml b/kubernetes/apps/cnpg-system/kustomization.yaml new file mode 100644 index 00000000..d0e03ab7 --- /dev/null +++ b/kubernetes/apps/cnpg-system/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./cloudnative-pg/ks.yaml diff --git a/kubernetes/apps/cnpg-system/namespace.yaml b/kubernetes/apps/cnpg-system/namespace.yaml new file mode 100644 index 00000000..1f89c520 --- /dev/null +++ b/kubernetes/apps/cnpg-system/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cnpg-system diff --git a/kubernetes/apps/kustomization.yaml b/kubernetes/apps/kustomization.yaml new file mode 100644 index 00000000..b45a1f5a --- /dev/null +++ b/kubernetes/apps/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./cnpg-system + - ./rook-ceph + - ./yucca diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/kubernetes/apps/rook-ceph/kustomization.yaml new file mode 100644 index 00000000..721753f8 --- /dev/null +++ b/kubernetes/apps/rook-ceph/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./rook-ceph-operator/ks.yaml + - ./rook-ceph-cluster/ks.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/kubernetes/apps/rook-ceph/namespace.yaml new file mode 100644 index 00000000..1696c56e --- /dev/null +++ b/kubernetes/apps/rook-ceph/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph diff --git a/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/helmrelease.yaml new file mode 100644 index 00000000..f001efce --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/helmrelease.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: rook-ceph-cluster + namespace: rook-ceph +spec: + interval: 1h + chart: + spec: + chart: charts/rook-ceph-cluster + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + # Dev defaults (single-node, single-replica, loopback OSD) live in the chart's + # values.yaml. This is DEV ONLY — prod object storage is a separate Ceph. + values: {} diff --git a/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-cluster/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph-cluster/ks.yaml b/kubernetes/apps/rook-ceph/rook-ceph-cluster/ks.yaml new file mode 100644 index 00000000..a2ea6292 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-cluster/ks.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: rook-ceph-cluster + namespace: flux-system +spec: + targetNamespace: rook-ceph + path: ./kubernetes/apps/rook-ceph/rook-ceph-cluster/app + prune: true + wait: true + interval: 1h + timeout: 15m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: rook-ceph-operator diff --git a/kubernetes/apps/rook-ceph/rook-ceph-operator/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph-operator/app/helmrelease.yaml new file mode 100644 index 00000000..dedfb7c1 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-operator/app/helmrelease.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: rook-ceph-operator + namespace: rook-ceph +spec: + interval: 1h + chart: + spec: + chart: rook-ceph + version: v1.20.0 + sourceRef: + kind: HelmRepository + name: rook-release + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + # DEV footprint: object store only needs the bucket provisioner, not CSI. + csi: + enableRbdDriver: false + enableCephfsDriver: false + enableDiscoveryDaemon: true + monitoring: + enabled: false + # The dev cluster's OSD runs on a loopback block device (k3d has no spare + # disk); Rook filters loop devices out of discovery unless this is set. + allowLoopDevices: true diff --git a/kubernetes/apps/rook-ceph/rook-ceph-operator/app/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph-operator/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-operator/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph-operator/ks.yaml b/kubernetes/apps/rook-ceph/rook-ceph-operator/ks.yaml new file mode 100644 index 00000000..6eaacb83 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph-operator/ks.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: rook-ceph-operator + namespace: flux-system +spec: + targetNamespace: rook-ceph + path: ./kubernetes/apps/rook-ceph/rook-ceph-operator/app + prune: true + wait: true + interval: 1h + timeout: 10m + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/apps/yucca/database/app/helmrelease.yaml b/kubernetes/apps/yucca/database/app/helmrelease.yaml new file mode 100644 index 00000000..0cfdceb5 --- /dev/null +++ b/kubernetes/apps/yucca/database/app/helmrelease.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-database + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/cnpg-cluster + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + # Dev-mirror posture: chart defaults (1 instance, 1Gi) keep this tree 1:1 + # with the Tilt deployment. TODO(prod): a prod overlay raises instances/storage. + values: {} diff --git a/kubernetes/apps/yucca/database/app/kustomization.yaml b/kubernetes/apps/yucca/database/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/database/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/database/ks.yaml b/kubernetes/apps/yucca/database/ks.yaml new file mode 100644 index 00000000..3a40a923 --- /dev/null +++ b/kubernetes/apps/yucca/database/ks.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-database + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-database + path: ./kubernetes/apps/yucca/database/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: cnpg-operator diff --git a/kubernetes/apps/yucca/kustomization.yaml b/kubernetes/apps/yucca/kustomization.yaml new file mode 100644 index 00000000..1b96de78 --- /dev/null +++ b/kubernetes/apps/yucca/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./database/ks.yaml + - ./object-user/ks.yaml + - ./mock-oidc/ks.yaml + - ./victoria-metrics/ks.yaml + - ./victoria-logs/ks.yaml + - ./michael/ks.yaml + - ./yucca-api/ks.yaml + - ./yucca-admin-api/ks.yaml + - ./web/ks.yaml diff --git a/kubernetes/apps/yucca/michael/app/helmrelease.yaml b/kubernetes/apps/yucca/michael/app/helmrelease.yaml new file mode 100644 index 00000000..a5c7f991 --- /dev/null +++ b/kubernetes/apps/yucca/michael/app/helmrelease.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-michael + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/michael + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + # Dev (Tilt) uses the chart defaults; prod overrides the image + secrets. + image: + repository: ghcr.io/immich-app/yucca/michael # TODO: confirm prod registry + tag: 0.0.1 + # TODO(prod): source JWT_SECRET / S3 creds from an ExternalSecret, not values diff --git a/kubernetes/apps/yucca/michael/app/kustomization.yaml b/kubernetes/apps/yucca/michael/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/michael/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/michael/ks.yaml b/kubernetes/apps/yucca/michael/ks.yaml new file mode 100644 index 00000000..b502e4bc --- /dev/null +++ b/kubernetes/apps/yucca/michael/ks.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-michael + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-michael + path: ./kubernetes/apps/yucca/michael/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: yucca-object-user diff --git a/kubernetes/apps/yucca/mock-oidc/app/helmrelease.yaml b/kubernetes/apps/yucca/mock-oidc/app/helmrelease.yaml new file mode 100644 index 00000000..10151ae2 --- /dev/null +++ b/kubernetes/apps/yucca/mock-oidc/app/helmrelease.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-mock-oidc + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/mock-oidc + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + # DEV ONLY. Prod uses a real IdP (e.g. Zitadel) — drop this release and + # point yucca-api/admin-api OIDC_* at the real issuer via ExternalSecrets. + fullnameOverride: yucca-mock-oidc + image: + repository: ghcr.io/immich-app/yucca/mock-oidc-provider # TODO: confirm prod registry + tag: 0.0.1 diff --git a/kubernetes/apps/yucca/mock-oidc/app/kustomization.yaml b/kubernetes/apps/yucca/mock-oidc/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/mock-oidc/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/mock-oidc/ks.yaml b/kubernetes/apps/yucca/mock-oidc/ks.yaml new file mode 100644 index 00000000..268c97db --- /dev/null +++ b/kubernetes/apps/yucca/mock-oidc/ks.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-mock-oidc + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-mock-oidc + path: ./kubernetes/apps/yucca/mock-oidc/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/apps/yucca/namespace.yaml b/kubernetes/apps/yucca/namespace.yaml new file mode 100644 index 00000000..bad731ab --- /dev/null +++ b/kubernetes/apps/yucca/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: yucca diff --git a/kubernetes/apps/yucca/object-user/app/helmrelease.yaml b/kubernetes/apps/yucca/object-user/app/helmrelease.yaml new file mode 100644 index 00000000..4c47f981 --- /dev/null +++ b/kubernetes/apps/yucca/object-user/app/helmrelease.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-object-user + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/ceph-objectuser + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: {} diff --git a/kubernetes/apps/yucca/object-user/app/kustomization.yaml b/kubernetes/apps/yucca/object-user/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/object-user/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/object-user/ks.yaml b/kubernetes/apps/yucca/object-user/ks.yaml new file mode 100644 index 00000000..a0705965 --- /dev/null +++ b/kubernetes/apps/yucca/object-user/ks.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-object-user + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-object-user + path: ./kubernetes/apps/yucca/object-user/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: rook-ceph-cluster diff --git a/kubernetes/apps/yucca/victoria-logs/app/helmrelease.yaml b/kubernetes/apps/yucca/victoria-logs/app/helmrelease.yaml new file mode 100644 index 00000000..d216d574 --- /dev/null +++ b/kubernetes/apps/yucca/victoria-logs/app/helmrelease.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-victoria-logs + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: victoria-logs-single + version: 0.11.32 + sourceRef: + kind: HelmRepository + name: victoriametrics + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + server: + fullnameOverride: victoria-logs + # Dev-mirror posture (Tilt installs these exact values). + # TODO(prod): enable + size persistence for the target cluster. + persistentVolume: + enabled: false diff --git a/kubernetes/apps/yucca/victoria-logs/app/kustomization.yaml b/kubernetes/apps/yucca/victoria-logs/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/victoria-logs/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/victoria-logs/ks.yaml b/kubernetes/apps/yucca/victoria-logs/ks.yaml new file mode 100644 index 00000000..7e7ffff1 --- /dev/null +++ b/kubernetes/apps/yucca/victoria-logs/ks.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-victoria-logs + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-victoria-logs + path: ./kubernetes/apps/yucca/victoria-logs/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/apps/yucca/victoria-metrics/app/helmrelease.yaml b/kubernetes/apps/yucca/victoria-metrics/app/helmrelease.yaml new file mode 100644 index 00000000..7e59c39d --- /dev/null +++ b/kubernetes/apps/yucca/victoria-metrics/app/helmrelease.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-victoria-metrics + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: victoria-metrics-single + version: 0.35.0 + sourceRef: + kind: HelmRepository + name: victoriametrics + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + server: + fullnameOverride: victoria-metrics + # Dev-mirror posture (Tilt installs these exact values). + # TODO(prod): enable + size persistence for the target cluster. + persistentVolume: + enabled: false diff --git a/kubernetes/apps/yucca/victoria-metrics/app/kustomization.yaml b/kubernetes/apps/yucca/victoria-metrics/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/victoria-metrics/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/victoria-metrics/ks.yaml b/kubernetes/apps/yucca/victoria-metrics/ks.yaml new file mode 100644 index 00000000..3e518bcb --- /dev/null +++ b/kubernetes/apps/yucca/victoria-metrics/ks.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-victoria-metrics + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-victoria-metrics + path: ./kubernetes/apps/yucca/victoria-metrics/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/apps/yucca/web/app/helmrelease.yaml b/kubernetes/apps/yucca/web/app/helmrelease.yaml new file mode 100644 index 00000000..d8606738 --- /dev/null +++ b/kubernetes/apps/yucca/web/app/helmrelease.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-web + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/web + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + image: + repository: ghcr.io/immich-app/yucca/web # TODO: confirm prod registry + tag: 0.0.1 + # YUCCA_API_URL/PUBLIC_API_URL default to the in-cluster yucca-api service, + # which is correct in prod too. TODO(prod): add ingress for external access. diff --git a/kubernetes/apps/yucca/web/app/kustomization.yaml b/kubernetes/apps/yucca/web/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/web/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/web/ks.yaml b/kubernetes/apps/yucca/web/ks.yaml new file mode 100644 index 00000000..8f98e005 --- /dev/null +++ b/kubernetes/apps/yucca/web/ks.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-web + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-web + path: ./kubernetes/apps/yucca/web/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: yucca-api diff --git a/kubernetes/apps/yucca/yucca-admin-api/app/helmrelease.yaml b/kubernetes/apps/yucca/yucca-admin-api/app/helmrelease.yaml new file mode 100644 index 00000000..b162c2c7 --- /dev/null +++ b/kubernetes/apps/yucca/yucca-admin-api/app/helmrelease.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-admin-api + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/yucca-admin-api + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + image: + repository: ghcr.io/immich-app/yucca/yucca-admin-api # TODO: confirm prod registry + tag: 0.0.1 + # TODO(prod): real OIDC_ADMIN_* URLs + secrets via ExternalSecret diff --git a/kubernetes/apps/yucca/yucca-admin-api/app/kustomization.yaml b/kubernetes/apps/yucca/yucca-admin-api/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/yucca-admin-api/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/yucca-admin-api/ks.yaml b/kubernetes/apps/yucca/yucca-admin-api/ks.yaml new file mode 100644 index 00000000..666ab718 --- /dev/null +++ b/kubernetes/apps/yucca/yucca-admin-api/ks.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-admin-api + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-admin-api + path: ./kubernetes/apps/yucca/yucca-admin-api/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: yucca-database + - name: yucca-mock-oidc diff --git a/kubernetes/apps/yucca/yucca-api/app/helmrelease.yaml b/kubernetes/apps/yucca/yucca-api/app/helmrelease.yaml new file mode 100644 index 00000000..409020b2 --- /dev/null +++ b/kubernetes/apps/yucca/yucca-api/app/helmrelease.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: yucca-api + namespace: yucca +spec: + interval: 1h + chart: + spec: + chart: charts/yucca-api + sourceRef: + kind: GitRepository + name: yucca + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + image: + repository: ghcr.io/immich-app/yucca/yucca-api # TODO: confirm prod registry + tag: 0.0.1 + # Disabled: the migration Job currently assumes the DEV image layout (it + # runs `pnpm install` + sql-tools from the source tree), which a dist-only + # prod image doesn't have. Re-enable once the Job has a prod-compatible + # command. Dev migrates at boot either way. + migration: + enabled: false + # TODO(prod): real OIDC issuer/redirect URLs, ingress, probes, replicas, + # and JWT/OIDC secrets via an ExternalSecret instead of static secretData. diff --git a/kubernetes/apps/yucca/yucca-api/app/kustomization.yaml b/kubernetes/apps/yucca/yucca-api/app/kustomization.yaml new file mode 100644 index 00000000..3f1edb8c --- /dev/null +++ b/kubernetes/apps/yucca/yucca-api/app/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/yucca/yucca-api/ks.yaml b/kubernetes/apps/yucca/yucca-api/ks.yaml new file mode 100644 index 00000000..fe27109a --- /dev/null +++ b/kubernetes/apps/yucca/yucca-api/ks.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: yucca-api + namespace: flux-system +spec: + targetNamespace: yucca + commonMetadata: + labels: + app.kubernetes.io/name: yucca-api + path: ./kubernetes/apps/yucca/yucca-api/app + prune: true + wait: true + interval: 1h + retryInterval: 2m + timeout: 5m + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: yucca-database + - name: yucca-mock-oidc + - name: yucca-michael diff --git a/kubernetes/bootstrap/README.md b/kubernetes/bootstrap/README.md new file mode 100644 index 00000000..2229455a --- /dev/null +++ b/kubernetes/bootstrap/README.md @@ -0,0 +1,61 @@ +# Bootstrap (fresh cluster → Flux-managed) + +One-time procedure to take an empty cluster to a self-reconciling Flux install +of this tree. Everything after step 3 is GitOps — no further kubectl needed. + +## 1. Install Flux + +```bash +flux check --pre +flux install # or the flux-operator, once we adopt it +``` + +## 2. Git auth (the repo is private) + +`kubernetes/flux/repos/git-yucca.yaml` references `secretRef: yucca-repo-auth`. +Create it before applying anything, using a read-only deploy key (preferred) +or a fine-grained PAT: + +```bash +# deploy key (read-only) — add the printed public key to the GitHub repo +flux create secret git yucca-repo-auth \ + --url=ssh://git@github.com/immich-app/yucca \ + --ssh-key-algorithm=ecdsa --ssh-ecdsa-curve=p521 + +# …or a fine-grained PAT over https +flux create secret git yucca-repo-auth \ + --url=https://github.com/immich-app/yucca \ + --username=git --password="$GITHUB_TOKEN" +``` + +If you use the ssh deploy key, switch `spec.url` in `git-yucca.yaml` to the +`ssh://git@github.com/...` form. + +## 3. Seed the sources + entrypoint from the local checkout + +The `cluster-repos`/`cluster-apps` Kustomizations pull from the `yucca` +GitRepository — which doesn't exist until something applies it. Break the +chicken-and-egg from your checkout: + +```bash +kubectl apply -k kubernetes/flux/repos # GitRepository + HelmRepositories +kubectl apply -k kubernetes/flux/cluster # cluster-repos -> cluster-apps +``` + +From here Flux owns the tree: it reconciles `kubernetes/flux/repos` (including +any future source changes) and `kubernetes/apps`. + +## 4. Watch it converge + +```bash +flux get kustomizations --watch +flux get helmreleases -A +``` + +Expected order: operators (`cnpg-operator`, `rook-ceph-operator`) → +`rook-ceph-cluster` → `yucca-database`/`yucca-object-user` → apps → `yucca-web`. + +> **Before pointing this at a real prod cluster:** the app HelmReleases still +> carry dev-mirror values and `TODO(prod)` markers (image registries/tags, +> OIDC endpoints, ingress, persistence, ExternalSecrets). `mock-oidc` and the +> Rook dev cluster are dev-only. See `kubernetes/README.md`. diff --git a/kubernetes/components/common/kustomization.yaml b/kubernetes/components/common/kustomization.yaml new file mode 100644 index 00000000..ccac14d5 --- /dev/null +++ b/kubernetes/components/common/kustomization.yaml @@ -0,0 +1,6 @@ +# Reusable Kustomize component. Reserved for cross-app concerns that should be +# applied uniformly (e.g. common labels, a shared ExternalSecret store ref). +# Consume from an app's app/kustomization.yaml via: +# components: [../../../../components/common] +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component diff --git a/kubernetes/flux/cluster/apps.yaml b/kubernetes/flux/cluster/apps.yaml new file mode 100644 index 00000000..ad33afb3 --- /dev/null +++ b/kubernetes/flux/cluster/apps.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-apps + namespace: flux-system +spec: + interval: 30m + path: ./kubernetes/apps + prune: true + wait: false + sourceRef: + kind: GitRepository + name: yucca + dependsOn: + - name: cluster-repos diff --git a/kubernetes/flux/cluster/kustomization.yaml b/kubernetes/flux/cluster/kustomization.yaml new file mode 100644 index 00000000..e3816aeb --- /dev/null +++ b/kubernetes/flux/cluster/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./repos.yaml + - ./apps.yaml diff --git a/kubernetes/flux/cluster/repos.yaml b/kubernetes/flux/cluster/repos.yaml new file mode 100644 index 00000000..1ce969c4 --- /dev/null +++ b/kubernetes/flux/cluster/repos.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-repos + namespace: flux-system +spec: + interval: 1h + path: ./kubernetes/flux/repos + prune: true + wait: true + sourceRef: + kind: GitRepository + name: yucca diff --git a/kubernetes/flux/repos/git-yucca.yaml b/kubernetes/flux/repos/git-yucca.yaml new file mode 100644 index 00000000..51056b1d --- /dev/null +++ b/kubernetes/flux/repos/git-yucca.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: yucca + namespace: flux-system +spec: + interval: 30m + url: https://github.com/immich-app/yucca + ref: + branch: main + # The repo is private: bootstrap creates this secret (a GitHub deploy key or + # fine-grained PAT) BEFORE applying this tree — see kubernetes/bootstrap/. + secretRef: + name: yucca-repo-auth + # Flux pulls only the GitOps manifests and the Helm charts they reference. + ignore: | + /* + !/kubernetes + !/charts diff --git a/kubernetes/flux/repos/helm-cloudnative-pg.yaml b/kubernetes/flux/repos/helm-cloudnative-pg.yaml new file mode 100644 index 00000000..3c4c31b1 --- /dev/null +++ b/kubernetes/flux/repos/helm-cloudnative-pg.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: cloudnative-pg + namespace: flux-system +spec: + interval: 1h + url: https://cloudnative-pg.github.io/charts diff --git a/kubernetes/flux/repos/helm-rook.yaml b/kubernetes/flux/repos/helm-rook.yaml new file mode 100644 index 00000000..4937e92e --- /dev/null +++ b/kubernetes/flux/repos/helm-rook.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: rook-release + namespace: flux-system +spec: + interval: 1h + url: https://charts.rook.io/release diff --git a/kubernetes/flux/repos/helm-victoriametrics.yaml b/kubernetes/flux/repos/helm-victoriametrics.yaml new file mode 100644 index 00000000..b4ce2503 --- /dev/null +++ b/kubernetes/flux/repos/helm-victoriametrics.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: victoriametrics + namespace: flux-system +spec: + interval: 1h + url: https://victoriametrics.github.io/helm-charts diff --git a/kubernetes/flux/repos/kustomization.yaml b/kubernetes/flux/repos/kustomization.yaml new file mode 100644 index 00000000..ea00e9bb --- /dev/null +++ b/kubernetes/flux/repos/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./git-yucca.yaml + - ./helm-cloudnative-pg.yaml + - ./helm-victoriametrics.yaml + - ./helm-rook.yaml diff --git a/packages/e2e/k3d/hostmap.cjs b/packages/e2e/k3d/hostmap.cjs new file mode 100644 index 00000000..da280a01 --- /dev/null +++ b/packages/e2e/k3d/hostmap.cjs @@ -0,0 +1,28 @@ +// e2e-against-k3d glue: the deployed yucca-api advertises its OIDC issuer as the +// in-cluster name http://yucca-mock-oidc:8092. Host-side fetch() (undici) in the +// jest e2e must resolve that name to the kubectl port-forward on localhost. +const dns = require('dns'); +const MAP = { 'yucca-mock-oidc': '127.0.0.1' }; +const origLookup = dns.lookup; +dns.lookup = function (hostname, options, callback) { + if (typeof options === 'function') { + callback = options; + options = {}; + } + if (MAP[hostname]) { + const rec = { address: MAP[hostname], family: 4 }; + if (options && options.all) return process.nextTick(callback, null, [rec]); + return process.nextTick(callback, null, rec.address, 4); + } + return origLookup.call(this, hostname, options || {}, callback); +}; +if (dns.promises && dns.promises.lookup) { + const origP = dns.promises.lookup; + dns.promises.lookup = async function (hostname, options) { + if (MAP[hostname]) { + const rec = { address: MAP[hostname], family: 4 }; + return options && options.all ? [rec] : rec; + } + return origP.call(this, hostname, options); + }; +} diff --git a/packages/e2e/k3d/run.sh b/packages/e2e/k3d/run.sh new file mode 100755 index 00000000..8b194dd6 --- /dev/null +++ b/packages/e2e/k3d/run.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Run the full e2e suite against the LOCAL k3d/Tilt stack. +# +# orchestration-api is intentionally NOT deployed to k8s — it runs here as a +# separate local process (per design). Everything else (yucca-api, michael, +# mock-oidc, web, db, Ceph) comes from the cluster via kubectl port-forwards. +# +# No /etc/hosts / sudo required: +# - the jest suite resolves the in-cluster OIDC issuer host via a dns preload +# (hostmap.cjs); the playwright browser via --host-resolver-rules. +# - yucca-api is told (e2e-only) to emit localhost restic URLs, because restic +# runs on this host (the chart keeps the in-cluster name for real use). +# +# Prereq: mise k3d:up && mise tilt:up (cluster healthy, context k3d-yucca) +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" +cd "$ROOT" + +[ "$(kubectl config current-context)" = "k3d-yucca" ] || { + echo "Not on k3d-yucca. Run: mise k3d:up && mise tilt:up" >&2; exit 1; } + +PF_PIDS=() +ORCH_PID="" +cleanup() { + echo "==> cleanup" + # Kill only what WE started (a broad pkill would take out the user's own + # port-forwards). The orchestrator is a mise->pnpm->node tree, so also match + # its exact child command. + [ -n "$ORCH_PID" ] && kill "$ORCH_PID" 2>/dev/null || true + pkill -f "pnpm --filter @futo-org/backups-orchestrator-api dev" 2>/dev/null || true + for p in "${PF_PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done + # Restore the chart default (in-cluster michael) on yucca-api. If this run is + # SIGKILLed the override survives — the next run reapplies + reverts it. + kubectl -n yucca set env deploy/yucca-api RESTIC_ENDPOINT- >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "==> build workspace libs (sdk consumed by orchestration-api + the e2e)" +mise run common:build >/dev/null +mise run yucca-sdk:orchestration-ui:build >/dev/null + +echo "==> port-forward k3d services to the e2e host ports" +kubectl port-forward -n yucca svc/yucca-michael 3010:3010 >/tmp/yucca-e2e-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n yucca svc/yucca-mock-oidc 8092:8092 >>/tmp/yucca-e2e-pf.log 2>&1 & PF_PIDS+=($!) +# web on :36033 (orchestration-api + the web e2e target) AND :5173 (yucca-api's +# OIDC redirect_uri, which the OIDC callback follows). +kubectl port-forward -n yucca svc/yucca-web 36033:5173 >>/tmp/yucca-e2e-pf.log 2>&1 & PF_PIDS+=($!) +kubectl port-forward -n yucca svc/yucca-web 5173:5173 >>/tmp/yucca-e2e-pf.log 2>&1 & PF_PIDS+=($!) + +echo "==> e2e-only: yucca-api emits localhost restic URLs (restic runs on this host)" +kubectl -n yucca set env deploy/yucca-api RESTIC_ENDPOINT="http://localhost:3010" >/dev/null +kubectl -n yucca rollout status deploy/yucca-api --timeout=120s >/dev/null +kubectl port-forward -n yucca svc/yucca-api 3020:3020 >>/tmp/yucca-e2e-pf.log 2>&1 & PF_PIDS+=($!) +sleep 5 + +echo "==> launch orchestration-api as a separate local process (:22676)" +NODE_OPTIONS="--require $HERE/hostmap.cjs" \ + mise run yucca-sdk:orchestration-api:dev >/tmp/yucca-e2e-orch.log 2>&1 & ORCH_PID=$! +for _ in $(seq 1 40); do + [ "$(curl -s -o /dev/null -w '%{http_code}' -m2 http://localhost:22676/ 2>/dev/null)" = "404" ] && break + sleep 2 +done + +echo "==> jest e2e (it-works, restic-api, yucca-api, orchestration-api)" +# shellcheck disable=SC1091 +source .mise/tasks/restic-api/env +# shellcheck disable=SC1091 +source .mise/tasks/yucca-api/env +NODE_OPTIONS="--experimental-vm-modules --require $HERE/hostmap.cjs" \ + pnpm --filter e2e exec jest --runInBand + +echo "==> web e2e (playwright against the k3d web)" +# Config lives in packages/web so @playwright/test resolves from web's deps. +pnpm --filter web exec playwright test --config="$ROOT/packages/web/playwright.k3d.config.ts" + +echo "==> ALL E2E PASSED against local k8s" diff --git a/packages/michael/.air.toml b/packages/michael/.air.toml new file mode 100644 index 00000000..f8a0fa97 --- /dev/null +++ b/packages/michael/.air.toml @@ -0,0 +1,9 @@ +root = "." +tmp_dir = "tmp" + +[build] + cmd = "go build -o ./tmp/michael ." + bin = "./tmp/michael" + include_ext = ["go"] + exclude_dir = ["tmp"] + delay = 500 diff --git a/packages/michael/Dockerfile b/packages/michael/Dockerfile index 29058362..1b742242 100644 --- a/packages/michael/Dockerfile +++ b/packages/michael/Dockerfile @@ -1,6 +1,19 @@ # michael Docker image -ARG ALPINE_VERSION=3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 +ARG ALPINE_VERSION=3.23 +# Pinned alpine runtime base. NOTE: this digest is alpine:3.23's, so it must NOT +# be appended to the node/golang base tags (which reuse ${ALPINE_VERSION}). +ARG ALPINE_IMAGE=alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 + +FROM golang:1.25-alpine${ALPINE_VERSION} AS dev +WORKDIR /src +RUN apk add --no-cache bash restic && \ + go install github.com/air-verse/air@latest +COPY packages/michael/go.mod packages/michael/go.sum ./ +RUN go mod download +COPY packages/michael/ ./ +EXPOSE 3010 +CMD ["air", "-c", ".air.toml"] FROM golang:1.25-alpine${ALPINE_VERSION} AS builder WORKDIR /app @@ -11,7 +24,7 @@ RUN go mod download COPY packages/michael/ ./ RUN CGO_ENABLED=0 go build -o /michael . -FROM alpine:${ALPINE_VERSION} +FROM ${ALPINE_IMAGE} RUN apk add --no-cache dumb-init \ && addgroup -g 1000 michael && adduser -u 1000 -G michael -s /bin/sh -D michael diff --git a/packages/web/Dockerfile b/packages/web/Dockerfile index 475e84e3..b47fbdc6 100644 --- a/packages/web/Dockerfile +++ b/packages/web/Dockerfile @@ -11,7 +11,45 @@ # https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md # https://pnpm.io/cli/deploy -ARG ALPINE_VERSION=3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 +ARG ALPINE_VERSION=3.23 +# Pinned alpine runtime base. NOTE: this digest is alpine:3.23's, so it must NOT +# be appended to the node/golang base tags (which reuse ${ALPINE_VERSION}). +ARG ALPINE_IMAGE=alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 + +# Manifest stage: keep only package.jsons + workspace files so the pnpm-install +# layer is cached on lockfile/manifest changes only (not every src edit). +FROM ${ALPINE_IMAGE} AS manifests +WORKDIR /src +COPY . ./ +RUN find . -type d -name node_modules -prune -exec rm -rf {} + && \ + find . -type f \ + ! -name 'package.json' \ + ! -name 'pnpm-lock.yaml' \ + ! -name 'pnpm-workspace.yaml' \ + ! -name '.npmrc' \ + -delete && \ + find . -type d -empty -delete + +FROM node:25-alpine${ALPINE_VERSION} AS dev +WORKDIR /app +RUN apk add --no-cache bash && npm install -g pnpm@10.28.1 +ENV NODE_ENV="development" + +# 1. Install deps — cached unless lockfile or any package.json changes +COPY --from=manifests /src ./ +RUN pnpm install --frozen-lockfile + +# 2. Copy sources and pre-build the workspace libs web consumes +COPY . ./ +# NOTE: filter by published package names (renamed under @futo-org/*); a stale +# filter silently matches nothing (pnpm exits 0), leaving web's SSR deps unbuilt. +RUN pnpm --filter @common/server build && \ + pnpm --filter @futo-org/backups-api-client build && \ + pnpm --filter @futo-org/backups-orchestrator-ui build && \ + pnpm --filter web lingui:compile + +EXPOSE 5173 +CMD ["pnpm", "--filter", "web", "dev", "--host", "0.0.0.0"] FROM node:25-alpine${ALPINE_VERSION} AS builder WORKDIR /build-stage @@ -34,7 +72,7 @@ RUN pnpm i --frozen-lockfile RUN NODE_ENV=production mise web:build RUN pnpm --filter web deploy /deploy --prod --legacy -FROM alpine:${ALPINE_VERSION} +FROM ${ALPINE_IMAGE} WORKDIR /usr/src/app diff --git a/packages/web/playwright.k3d.config.ts b/packages/web/playwright.k3d.config.ts new file mode 100644 index 00000000..fb30378d --- /dev/null +++ b/packages/web/playwright.k3d.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from '@playwright/test'; + +// Web e2e against the LOCAL k3d stack (web port-forwarded to :36033). Unlike the +// default config there is no webServer block — web is already served by the +// cluster. host-resolver-rules lets the browser reach the in-cluster OIDC issuer +// host (yucca-mock-oidc) via the kubectl port-forward on localhost. Driven by +// packages/e2e/k3d/run.sh (mise test:e2e:k3d). +export default defineConfig({ + testDir: 'e2e', + use: { + launchOptions: { + args: ['--host-resolver-rules=MAP yucca-mock-oidc 127.0.0.1'], + }, + }, +}); diff --git a/packages/web/vite.config.ts b/packages/web/vite.config.ts index 0e9878bf..7096f44a 100644 --- a/packages/web/vite.config.ts +++ b/packages/web/vite.config.ts @@ -14,7 +14,9 @@ export default defineConfig({ server: { proxy: { '/api': { - target: `http://localhost:${process.env.YUCCA_API_PORT}/`, + target: + process.env.YUCCA_API_URL ?? + `http://localhost:${process.env.YUCCA_API_PORT}/`, secure: true, changeOrigin: true, ws: true, diff --git a/packages/yucca-admin-api/Dockerfile b/packages/yucca-admin-api/Dockerfile index 045c47ce..a036e367 100644 --- a/packages/yucca-admin-api/Dockerfile +++ b/packages/yucca-admin-api/Dockerfile @@ -1,4 +1,4 @@ -# yucca-api Docker image +# yucca-admin-api Docker image # Built on/for Alpine Linux # # mise is used as a command runner only @@ -12,7 +12,40 @@ # https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md # https://pnpm.io/cli/deploy -ARG ALPINE_VERSION=3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 +ARG ALPINE_VERSION=3.23 +# Pinned alpine runtime base. NOTE: this digest is alpine:3.23's, so it must NOT +# be appended to the node/golang base tags (which reuse ${ALPINE_VERSION}). +ARG ALPINE_IMAGE=alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 + +# Manifest stage: strips everything except package.jsons + workspace files so +# the pnpm-install layer below is cached on lockfile/manifest changes only. +FROM ${ALPINE_IMAGE} AS manifests +WORKDIR /src +COPY . ./ +RUN find . -type d -name node_modules -prune -exec rm -rf {} + && \ + find . -type f \ + ! -name 'package.json' \ + ! -name 'pnpm-lock.yaml' \ + ! -name 'pnpm-workspace.yaml' \ + ! -name '.npmrc' \ + -delete && \ + find . -type d -empty -delete + +FROM node:25-alpine${ALPINE_VERSION} AS dev +WORKDIR /app +RUN apk add --no-cache bash && npm install -g pnpm@10.28.1 +ENV NODE_ENV="development" + +# 1. Install deps — cached unless lockfile or any package.json changes +COPY --from=manifests /src ./ +RUN pnpm install --frozen-lockfile + +# 2. Copy sources, build workspace libs yucca-admin-api imports at runtime +COPY . ./ +RUN pnpm --filter @common/server build + +EXPOSE 3030 +CMD ["pnpm", "--filter", "yucca-admin-api", "start:dev"] FROM node:25-alpine${ALPINE_VERSION} AS builder WORKDIR /build-stage @@ -33,10 +66,10 @@ RUN npm install -g pnpm@10.28.1 RUN curl https://mise.run | sh RUN mise trust RUN pnpm i --frozen-lockfile -RUN mise yucca-api:build -RUN pnpm --filter yucca-api deploy /deploy --prod --legacy +RUN mise yucca-admin-api:build +RUN pnpm --filter yucca-admin-api deploy /deploy --prod --legacy -FROM alpine:${ALPINE_VERSION} +FROM ${ALPINE_IMAGE} WORKDIR /usr/src/app @@ -51,5 +84,5 @@ USER node COPY --from=builder /deploy ./ ENV NODE_ENV="production" -EXPOSE 3020 +EXPOSE 3030 CMD ["dumb-init", "node", "dist/main"] diff --git a/packages/yucca-api/Dockerfile b/packages/yucca-api/Dockerfile index 045c47ce..49e2a953 100644 --- a/packages/yucca-api/Dockerfile +++ b/packages/yucca-api/Dockerfile @@ -12,7 +12,40 @@ # https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md # https://pnpm.io/cli/deploy -ARG ALPINE_VERSION=3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 +ARG ALPINE_VERSION=3.23 +# Pinned alpine runtime base. NOTE: this digest is alpine:3.23's, so it must NOT +# be appended to the node/golang base tags (which reuse ${ALPINE_VERSION}). +ARG ALPINE_IMAGE=alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 + +# Manifest stage: strips everything except package.jsons + workspace files so +# the pnpm-install layer below is cached on lockfile/manifest changes only. +FROM ${ALPINE_IMAGE} AS manifests +WORKDIR /src +COPY . ./ +RUN find . -type d -name node_modules -prune -exec rm -rf {} + && \ + find . -type f \ + ! -name 'package.json' \ + ! -name 'pnpm-lock.yaml' \ + ! -name 'pnpm-workspace.yaml' \ + ! -name '.npmrc' \ + -delete && \ + find . -type d -empty -delete + +FROM node:25-alpine${ALPINE_VERSION} AS dev +WORKDIR /app +RUN apk add --no-cache bash && npm install -g pnpm@10.28.1 +ENV NODE_ENV="development" + +# 1. Install deps — cached unless lockfile or any package.json changes +COPY --from=manifests /src ./ +RUN pnpm install --frozen-lockfile + +# 2. Copy sources, build workspace libs yucca-api imports at runtime +COPY . ./ +RUN pnpm --filter @common/server build + +EXPOSE 3020 +CMD ["pnpm", "--filter", "yucca-api", "start:dev"] FROM node:25-alpine${ALPINE_VERSION} AS builder WORKDIR /build-stage @@ -36,7 +69,7 @@ RUN pnpm i --frozen-lockfile RUN mise yucca-api:build RUN pnpm --filter yucca-api deploy /deploy --prod --legacy -FROM alpine:${ALPINE_VERSION} +FROM ${ALPINE_IMAGE} WORKDIR /usr/src/app