diff --git a/kubernetes/apps/staging/austin/observability/kustomization.yaml b/kubernetes/apps/staging/austin/observability/kustomization.yaml index 0b14a8a5..05cd9bee 100644 --- a/kubernetes/apps/staging/austin/observability/kustomization.yaml +++ b/kubernetes/apps/staging/austin/observability/kustomization.yaml @@ -1,8 +1,71 @@ --- # staging@austin observability overlay — the shared components/observability -# slice, unpatched: staging keeps the authed public futostatus egress the base -# manifests ship (father repoints at o11y's mesh vmauth in its overlay). +# slice, with the remote-write egress repointed at o11y STAGING's mesh vmauth +# over NetBird (vmauth.staging.o11y.futo.network → 10.69.1.10, unauthenticated: +# the yucca-side talos-to-o11y-gateway NetBird ACL is the only gate; the +# mesh-unauth vmauth 401s requests that DO carry a bearer, so the token wiring +# must go, not just be ignored). Mirrors father's mesh path. +# +# The mesh name is pinned pod-side (hostAliases): the NetBird DNS zone is not +# distributed to the staging talos peers, and 10.69.1.10 is the +# netbox-registered gateway VIP (cidrhost of o11y staging's service CIDR). apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../../components/observability +patches: + # vmagent: mesh URL, no bearer, pinned mesh name. + - target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + name: vmagent + patch: |- + - op: add + path: /spec/patches + value: + - target: + kind: HelmRelease + name: vmagent + patch: |- + - op: replace + path: /spec/values/vmagent/spec/remoteWrite/0/url + value: https://vmauth.staging.o11y.futo.network/insert/0/prometheus/api/v1/write + - op: remove + path: /spec/values/vmagent/spec/remoteWrite/0/bearerTokenSecret + - op: add + path: /spec/values/vmagent/spec/hostAliases + value: + - ip: 10.69.1.10 + hostnames: + - vmauth.staging.o11y.futo.network + # logs collector: URL comes from VLOGS_REMOTE_URL (cluster-settings); the + # bearer flag AND its token volume go (the vmagent-remote-write Secret is no + # longer provisioned), and the mesh name is pinned via postRenderers — the + # chart has no hostAliases value. + - target: + group: kustomize.toolkit.fluxcd.io + kind: Kustomization + name: victoria-logs-collector + patch: |- + - op: add + path: /spec/patches + value: + - target: + kind: HelmRelease + name: victoria-logs-collector + patch: |- + - op: remove + path: /spec/values/extraArgs/remoteWrite.bearerTokenFile + - op: remove + path: /spec/values/extraVolumes + - op: remove + path: /spec/values/extraVolumeMounts + - op: add + path: /spec/postRenderers + value: + - kustomize: + patches: + - target: + kind: DaemonSet + name: victoria-logs-collector + patch: '[{"op": "add", "path": "/spec/template/spec/hostAliases", "value": [{"ip": "10.69.1.10", "hostnames": ["vmauth.staging.o11y.futo.network"]}]}]' diff --git a/kubernetes/clusters/staging/austin/cluster-settings.generated.yaml b/kubernetes/clusters/staging/austin/cluster-settings.generated.yaml index a92b141c..80560198 100644 --- a/kubernetes/clusters/staging/austin/cluster-settings.generated.yaml +++ b/kubernetes/clusters/staging/austin/cluster-settings.generated.yaml @@ -41,8 +41,9 @@ data: S3_ENDPOINT: https://s3.staging.austin.int.futo.cloud S3_HOST: s3.staging.austin.int.futo.cloud - # Observability egress (apps -> agents -> o11y vmauth). In-cluster OTLP - # receiver for metrics (host:port, no scheme); VictoriaLogs native ingest URL - # for the collector's egress. + # Observability egress (apps -> agents -> o11y STAGING's mesh vmauth over + # NetBird, unauthenticated — the talos-to-o11y-gateway ACL is the gate; see + # apps/staging/austin/observability). In-cluster OTLP receiver for metrics + # (host:port, no scheme); VictoriaLogs native ingest URL for the collector. VMAGENT_OTLP: vmagent-yucca.observability.svc:8429 - VLOGS_REMOTE_URL: https://vmauth.staging.futostatus.com/insert/native + VLOGS_REMOTE_URL: https://vmauth.staging.o11y.futo.network/insert/native diff --git a/tf/.env b/tf/.env index 4ae308e2..1079dbb7 100644 --- a/tf/.env +++ b/tf/.env @@ -61,9 +61,6 @@ export TF_VAR_sietch_db_backup_access_key="op://yucca_tf_staging/SIETCH_CEPH_S3_ export TF_VAR_sietch_db_backup_secret_key="op://yucca_tf_staging/SIETCH_CEPH_S3_SVC_YUCCA_DB_BACKUP_SECRET_KEY/password" export TF_VAR_sietch_rgw_tls_cert="op://yucca_tf_staging/SIETCH_CEPH_RGW_TLS_CERT/password" -# vmagent + collector → o11y staging vmauth bearer token. -export TF_VAR_vmauth_remote_write_password="op://shared_tf_staging/VICTORIAMETRICS_VMAUTH_PASSWORD/password" - # Cloudflare API token for cert-manager DNS-01 (same item, TF_VAR form). export TF_VAR_cloudflare_api_token="op://yucca_tf_staging/CLOUDFLARE_API_TOKEN/password" diff --git a/tf/deployment/staging/austin/talos/secrets.tf b/tf/deployment/staging/austin/talos/secrets.tf index 3468e2e6..131fe873 100644 --- a/tf/deployment/staging/austin/talos/secrets.tf +++ b/tf/deployment/staging/austin/talos/secrets.tf @@ -229,19 +229,6 @@ resource "kubernetes_secret_v1" "yucca_db_backup_s3" { } } -# ─── Observability Secret (namespace: observability) ──────────────────── -# Bearer token vmagent + vlagent present to o11y's vmauth for remote-write. -resource "kubernetes_secret_v1" "vmagent_remote_write" { - count = local.provision_secrets ? 1 : 0 - metadata { - name = "vmagent-remote-write" - namespace = kubernetes_namespace_v1.observability[0].metadata[0].name - } - data = { - token = var.vmauth_remote_write_password - } -} - # ─── cert-manager Secret (namespace: cert-manager) ────────────────────── # Cloudflare API token for the Let's Encrypt DNS-01 ClusterIssuer (same 1P # item the dns stack uses — Zone:Read + DNS:Edit on futo.cloud). diff --git a/tf/deployment/staging/austin/talos/variables.tf b/tf/deployment/staging/austin/talos/variables.tf index b04d95f3..95cfbec2 100644 --- a/tf/deployment/staging/austin/talos/variables.tf +++ b/tf/deployment/staging/austin/talos/variables.tf @@ -135,16 +135,6 @@ variable "sietch_rgw_tls_cert" { default = "" } -# Bearer token vmagent uses to remote-write metrics to o11y's vmauth. This is -# the shared VICTORIAMETRICS_VMAUTH_PASSWORD from the shared_tf_staging vault (the -# `remote-clusters` VMUser authenticates remote clusters with it). -variable "vmauth_remote_write_password" { - description = "o11y vmauth bearer token for vmagent remote-write. Injected via TF_VAR from 1P (shared_tf_staging/VICTORIAMETRICS_VMAUTH_PASSWORD)." - type = string - sensitive = true - default = "" -} - # Cloudflare API token for the cert-manager DNS-01 ClusterIssuer (futo.cloud # zone). Same 1P item the dns stack uses. Injected via TF_VAR from 1P. variable "cloudflare_api_token" { diff --git a/tf/deployment/staging/global/netbird/main.tf b/tf/deployment/staging/global/netbird/main.tf index 4cff7f40..7b6c3dbb 100644 --- a/tf/deployment/staging/global/netbird/main.tf +++ b/tf/deployment/staging/global/netbird/main.tf @@ -31,6 +31,15 @@ data "netbird_group" "lp_services" { name = "Liberty Park Services" } +# o11y's staging mesh gateway group (owned by the yucca-o11y repo's netbird TF) +# — destination of the talos-to-o11y-gateway policy (netbird.auto.tfvars): the +# logs collector remote-writes to the mesh vmauth +# (vmauth.staging.o11y.futo.network → the gateway VIP behind o11y's routing +# peers). Mirrors prod/htz-fsn1/netbird. +data "netbird_group" "o11y_k8s_gateway" { + name = "o11y-staging-k8s-gateway" +} + module "netbird" { source = "../../../../shared/modules/netbird-env" @@ -47,6 +56,7 @@ module "netbird" { lp_server_monitoring = data.netbird_group.lp_server_monitoring.id lp_servers = data.netbird_group.lp_servers.id lp_services = data.netbird_group.lp_services.id + o11y_k8s_gateway = data.netbird_group.o11y_k8s_gateway.id } } diff --git a/tf/deployment/staging/global/netbird/netbird.auto.tfvars b/tf/deployment/staging/global/netbird/netbird.auto.tfvars index df642557..f0dffc59 100644 --- a/tf/deployment/staging/global/netbird/netbird.auto.tfvars +++ b/tf/deployment/staging/global/netbird/netbird.auto.tfvars @@ -32,6 +32,22 @@ policies = { }] } + # Talos nodes → o11y's staging mesh gateway (external group, resolved in + # main.tf): vmagent + victoria-logs-collector remote-write to the + # UNAUTHENTICATED mesh vmauth (vmauth.staging.o11y.futo.network:443) — this + # ACL is the only gate. Mirrors prod's talos-to-o11y-gateway. + talos-to-o11y-gateway = { + description = "Talos nodes → o11y staging mesh gateway (unauth vmauth remote-write)." + rules = [{ + name = "talos-to-o11y-gateway" + protocol = "tcp" + bidirectional = false + sources = ["talos"] + destinations = ["o11y_k8s_gateway"] + ports = ["443"] + }] + } + # CI reaches the existing Liberty Park infra groups where the staging nodes # live today (the targets CI talks to over the overlay). lp_* are external # groups resolved by name in main.tf.