From 114904e9662eba00cd0cacb590c4b5ddea8b8536 Mon Sep 17 00:00:00 2001 From: Antoine Lecompte <38678863+nutgood@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:09:30 -0400 Subject: [PATCH] feat(prod): continue prod (#288) --- .../templates/deployment.yaml | 7 ++- .../apps/base/envoy-proxy/certificate.yaml | 10 ++-- kubernetes/apps/base/envoy-proxy/gateway.yaml | 2 +- kubernetes/apps/prod/htz-fsn1/coredns.yaml | 3 +- .../apps/prod/htz-fsn1/gw-proxy/gateway.yaml | 2 +- .../apps/prod/htz-fsn1/netops/vmagent.yaml | 6 ++- .../htz-fsn1/observability/kustomization.yaml | 48 +++++++++++++++++++ .../prod/htz-fsn1/platform/observability.yaml | 29 +++++++++-- .../htz-fsn1/cluster-settings.generated.yaml | 5 +- .../prod/htz-fsn1/netbird/netbird.auto.tfvars | 16 +++++++ .../prod/htz-fsn1/netbird/netbird.tf | 17 +++++-- 11 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 kubernetes/apps/prod/htz-fsn1/observability/kustomization.yaml diff --git a/charts/apps/yucca-metrics-worker/templates/deployment.yaml b/charts/apps/yucca-metrics-worker/templates/deployment.yaml index f2234909..8ee304e9 100644 --- a/charts/apps/yucca-metrics-worker/templates/deployment.yaml +++ b/charts/apps/yucca-metrics-worker/templates/deployment.yaml @@ -9,8 +9,11 @@ (dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password"))) )) }} {{- /* extraEnvFrom comes after the chart secret: for duplicate keys Kubernetes -takes the LAST envFrom source, so these act as overrides. */}} +takes the LAST envFrom source, so these act as overrides. optional: everything +the worker needs is explicit env above — deployments that null secretData +(staging/prod) provide no Secret at all, and Optional=false would wedge the +pod in CreateContainerConfigError. */}} {{- $_ := set .Values "envFrom" (concat - (list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .)))) + (list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .) "optional" true))) (.Values.extraEnvFrom | default (list))) }} {{- include "yucca-common.deployment" . }} diff --git a/kubernetes/apps/base/envoy-proxy/certificate.yaml b/kubernetes/apps/base/envoy-proxy/certificate.yaml index 2710608b..d473da13 100644 --- a/kubernetes/apps/base/envoy-proxy/certificate.yaml +++ b/kubernetes/apps/base/envoy-proxy/certificate.yaml @@ -1,12 +1,12 @@ --- # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/cert-manager.io/certificate_v1.json -# Wildcard + apex for the staging domain — covers web (apex), api., gw. (and -# admin. when it's exposed). cert-manager writes staging-backups-tls into this -# namespace (envoy-system) for the Gateway to terminate with. +# Wildcard + apex for the cluster's APP_DOMAIN — covers web (apex), api., gw. +# (and admin. when it's exposed). cert-manager writes app-domain-tls into this +# namespace (envoy-system) for the Gateway(s) to terminate with. apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: staging-backups + name: app-domain spec: dnsNames: - "${APP_DOMAIN}" @@ -19,4 +19,4 @@ spec: issuerRef: kind: ClusterIssuer name: letsencrypt-production - secretName: staging-backups-tls + secretName: app-domain-tls diff --git a/kubernetes/apps/base/envoy-proxy/gateway.yaml b/kubernetes/apps/base/envoy-proxy/gateway.yaml index 8e52ec9f..70c033a6 100644 --- a/kubernetes/apps/base/envoy-proxy/gateway.yaml +++ b/kubernetes/apps/base/envoy-proxy/gateway.yaml @@ -22,7 +22,7 @@ spec: mode: Terminate certificateRefs: - kind: Secret - name: staging-backups-tls + name: app-domain-tls - name: http protocol: HTTP port: 80 diff --git a/kubernetes/apps/prod/htz-fsn1/coredns.yaml b/kubernetes/apps/prod/htz-fsn1/coredns.yaml index 85903be8..3aff7170 100644 --- a/kubernetes/apps/prod/htz-fsn1/coredns.yaml +++ b/kubernetes/apps/prod/htz-fsn1/coredns.yaml @@ -69,7 +69,8 @@ data: Corefile: ".:53 {\n errors\n health {\n lameduck 5s\n }\n ready\n\ \ log . {\n class error\n }\n prometheus :9153\n\n kubernetes\ \ cluster.local in-addr.arpa ip6.arpa {\n pods insecure\n fallthrough\ - \ in-addr.arpa ip6.arpa\n ttl 30\n }\n forward . /etc/resolv.conf\ + \ in-addr.arpa ip6.arpa\n ttl 30\n }\n hosts {\n 10.69.0.10\ + \ vmauth.o11y.futo.network\n fallthrough\n }\n forward . /etc/resolv.conf\ \ {\n max_concurrent 1000\n }\n cache 30 {\n disable success\ \ cluster.local\n disable denial cluster.local\n }\n loop\n reload\n\ \ loadbalance\n}\n" diff --git a/kubernetes/apps/prod/htz-fsn1/gw-proxy/gateway.yaml b/kubernetes/apps/prod/htz-fsn1/gw-proxy/gateway.yaml index 97360c06..177d5dba 100644 --- a/kubernetes/apps/prod/htz-fsn1/gw-proxy/gateway.yaml +++ b/kubernetes/apps/prod/htz-fsn1/gw-proxy/gateway.yaml @@ -27,4 +27,4 @@ spec: mode: Terminate certificateRefs: - kind: Secret - name: staging-backups-tls + name: app-domain-tls diff --git a/kubernetes/apps/prod/htz-fsn1/netops/vmagent.yaml b/kubernetes/apps/prod/htz-fsn1/netops/vmagent.yaml index 0ab9db59..b02e7c4e 100644 --- a/kubernetes/apps/prod/htz-fsn1/netops/vmagent.yaml +++ b/kubernetes/apps/prod/htz-fsn1/netops/vmagent.yaml @@ -142,8 +142,10 @@ spec: - { name: config, mountPath: /config } - { name: buffer, mountPath: /buffer } resources: - requests: { cpu: 50m, memory: 128Mi } - limits: { memory: 512Mi } + requests: { cpu: 50m, memory: 256Mi } + # 512Mi OOM-looped every ~8min once the 12 k8s + 5 static targets + # were all scraping (269 restarts over 6 days). + limits: { memory: 4Gi } volumes: - name: config configMap: { name: vmagent-config } diff --git a/kubernetes/apps/prod/htz-fsn1/observability/kustomization.yaml b/kubernetes/apps/prod/htz-fsn1/observability/kustomization.yaml new file mode 100644 index 00000000..d6b59d21 --- /dev/null +++ b/kubernetes/apps/prod/htz-fsn1/observability/kustomization.yaml @@ -0,0 +1,48 @@ +--- +# prod@htz-fsn1 observability overlay — the shared components/infra/observability +# slice, with the remote-write egress repointed at o11y's MESH vmauth +# (vmauth.o11y.futo.network → 10.69.0.10 over NetBird, unauthenticated: the +# NetBird ACL `yucca-prod-htz-fsn1-talos → o11y-production-k8s-gateway:443` is +# the only gate; the mesh-unauth vmauth 401s requests that DO carry a bearer, +# so the token wiring must go, not just be ignored). Staging keeps the authed +# public futostatus path. Pods resolve the mesh name via the coredns hosts +# entry (../coredns.yaml) — the NetBird DNS zone is not distributed to nodes. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../components/infra/observability +patches: + # vmagent: mesh URL, no bearer. + - target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + name: vmagent + patch: |- + - op: add + path: /spec/patches + value: + - target: + kind: HelmRelease + name: vmagent + patch: |- + - op: replace + path: /spec/values/vmagent/spec/remoteWrite/0/url + value: https://vmauth.o11y.futo.network/insert/0/prometheus/api/v1/write + - op: remove + path: /spec/values/vmagent/spec/remoteWrite/0/bearerTokenSecret + # logs collector: URL comes from VLOGS_REMOTE_URL (cluster-settings); only the + # bearer must go. + - target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + name: victoria-logs-collector + patch: |- + - op: add + path: /spec/patches + value: + - target: + kind: HelmRelease + name: victoria-logs-collector + patch: |- + - op: remove + path: /spec/values/extraArgs/remoteWrite.bearerTokenFile diff --git a/kubernetes/apps/prod/htz-fsn1/platform/observability.yaml b/kubernetes/apps/prod/htz-fsn1/platform/observability.yaml index dd2b706c..b9bad59f 100644 --- a/kubernetes/apps/prod/htz-fsn1/platform/observability.yaml +++ b/kubernetes/apps/prod/htz-fsn1/platform/observability.yaml @@ -1,9 +1,8 @@ # yaml-language-server: $schema=https://k8s-schemas.home-operations.com/kustomize.toolkit.fluxcd.io/kustomization_v1.json --- # Observability agents (vmagent + victoria-logs-collector) — the shared -# components/infra/observability slice (namespace, netpols, the two nested -# Kustomizations), reused as a plain kustomize path. The vmagent-remote-write -# Secret is TF-provisioned (talos stack secrets.tf). +# components/infra/observability slice via the prod overlay (../observability), +# which repoints remote-write at o11y's mesh vmauth over NetBird, unauth. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -13,10 +12,32 @@ spec: interval: 1h retryInterval: 2m timeout: 10m - path: ./kubernetes/components/infra/observability + path: ./kubernetes/apps/prod/htz-fsn1/observability prune: true wait: true sourceRef: kind: GitRepository name: flux-system namespace: flux-system + # The nested vmagent/victoria-logs-collector Kustomizations are applied by + # THIS Kustomization, not cluster-apps, so its substituteFrom patch doesn't + # reach them — re-apply it here or their ${vars} ship unsubstituted (the + # 17h victoria-logs-collector crash loop on father). + patches: + - target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + patch: |- + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: _ + spec: + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-settings-generated + - kind: ConfigMap + name: cluster-settings + - kind: ConfigMap + name: image-versions diff --git a/kubernetes/clusters/prod/htz-fsn1/cluster-settings.generated.yaml b/kubernetes/clusters/prod/htz-fsn1/cluster-settings.generated.yaml index 354879c2..86470862 100644 --- a/kubernetes/clusters/prod/htz-fsn1/cluster-settings.generated.yaml +++ b/kubernetes/clusters/prod/htz-fsn1/cluster-settings.generated.yaml @@ -26,6 +26,7 @@ data: S3_ENDPOINT: https://s3.prod.fsn1.htz.futo.cloud S3_HOST: s3.prod.fsn1.htz.futo.cloud - # Observability egress (apps -> agents -> o11y prod vmauth). + # Observability egress (apps -> agents -> o11y's MESH vmauth over NetBird, + # unauthenticated; see apps/prod/htz-fsn1/observability). VMAGENT_OTLP: vmagent-yucca.observability.svc:8429 - VLOGS_REMOTE_URL: https://vmauth.prod.futostatus.com/insert/native + VLOGS_REMOTE_URL: https://vmauth.o11y.futo.network/insert/native diff --git a/tf/deployment/prod/htz-fsn1/netbird/netbird.auto.tfvars b/tf/deployment/prod/htz-fsn1/netbird/netbird.auto.tfvars index 815aa67b..dfa535a4 100644 --- a/tf/deployment/prod/htz-fsn1/netbird/netbird.auto.tfvars +++ b/tf/deployment/prod/htz-fsn1/netbird/netbird.auto.tfvars @@ -106,6 +106,22 @@ policies = { }] } + # Talos nodes → o11y's prod mesh gateway (external group, resolved in + # netbird.tf): vmagent + victoria-logs-collector remote-write to the + # UNAUTHENTICATED mesh vmauth (vmauth.o11y.futo.network:443) — this ACL is + # the only gate. Mirrors o11y's own bootstrap-egress precedent. + talos-to-o11y-gateway = { + description = "Talos nodes → o11y prod mesh gateway (unauth vmauth remote-write)." + rules = [{ + name = "talos-to-o11y-gateway" + protocol = "tcp" + bidirectional = false + sources = ["talos"] + destinations = ["o11y_k8s_gateway"] + ports = ["443"] + }] + } + # Talos nodes reach the routed site subnets (esp. the kube fabric net 10.40.10/24) # via the mgmt route peers — this is how the cloud CPs' apiserver reaches the # bare-metal worker kubelets. diff --git a/tf/deployment/prod/htz-fsn1/netbird/netbird.tf b/tf/deployment/prod/htz-fsn1/netbird/netbird.tf index 91143ad8..fb8d59bd 100644 --- a/tf/deployment/prod/htz-fsn1/netbird/netbird.tf +++ b/tf/deployment/prod/htz-fsn1/netbird/netbird.tf @@ -75,6 +75,14 @@ locals { } } +# o11y's prod mesh gateway group (owned by the yucca-o11y repo's netbird TF) — +# destination of the talos-to-o11y-gateway policy (netbird.auto.tfvars): the +# observability agents remote-write to the mesh vmauth +# (vmauth.o11y.futo.network → the gateway VIP behind o11y's routing peers). +data "netbird_group" "o11y_k8s_gateway" { + name = "o11y-production-k8s-gateway" +} + module "netbird" { source = "../../../../shared/modules/netbird-env" @@ -82,10 +90,11 @@ module "netbird" { name_prefix = "yucca_${var.partition}_${var.region}" # yucca_prod_htz_fsn1 (slug normalized in the module) vault = "yucca_tf_${var.partition}" # yucca_tf_prod - groups = var.groups - setup_keys = var.setup_keys - policies = var.policies - networks = local.netbird_networks + groups = var.groups + external_groups = { o11y_k8s_gateway = data.netbird_group.o11y_k8s_gateway.id } + setup_keys = var.setup_keys + policies = var.policies + networks = local.netbird_networks } output "group_ids" {