feat(staging): normalize logs (#164)

kubernetes/apps/base/michael/helmrelease.yaml

@@ -36,8 +36,8 @@ spec:
         hostnames:
           - s3.dev.austin.int.futo.cloud
     # Full replacement of the chart's dev env: no Rook secretKeyRefs (S3 creds
-    # arrive via envFrom from the TF Secret), real RGW endpoint, OTLP to the
-    # in-cluster agents (vmagent metrics, Vector logs).
+    # arrive via envFrom from the TF Secret), real RGW endpoint, OTLP metrics to
+    # vmagent (logs are tailed from stdout by victoria-logs-collector).
     env:
       - name: RESTIC_API_PORT
         value: "3010"
@@ -53,7 +53,3 @@ spec:
         value: ${VMAGENT_OTLP}
       - name: OTLP_METRICS_URL_PATH
         value: /opentelemetry/v1/metrics
-      - name: OTLP_LOGS_ENDPOINT
-        value: ${VLAGENT_OTLP}
-      - name: OTLP_LOGS_URL_PATH
-        value: /insert/opentelemetry/v1/logs

kubernetes/apps/base/victoria-logs-collector/helmrelease.yaml

@@ -0,0 +1,57 @@
+---
+# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: victoria-logs-collector
+spec:
+  chartRef:
+    kind: OCIRepository
+    name: victoria-logs-collector
+  interval: 1h
+  rollback:
+    cleanupOnFail: true
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 2
+  values:
+    # DaemonSet that tails every pod's stdout node-side and forwards to o11y's
+    # VictoriaLogs. Replaces the OTLP-push vlagent: captures all pods (not just
+    # apps) and parses their JSON logs into fields.
+    fullnameOverride: victoria-logs-collector
+
+    # Ship to o11y's vmauth (native VictoriaLogs ingest -> vlinsert), authed with
+    # the shared bearer token (TF-provisioned `vmagent-remote-write` Secret).
+    remoteWrite:
+      - url: ${VLOGS_REMOTE_URL}
+        maxDiskUsagePerURL: 10GB
+    extraArgs:
+      remoteWrite.bearerTokenFile: /secrets/token
+    extraVolumes:
+      - name: token
+        secret:
+          secretName: vmagent-remote-write
+          items:
+            - key: token
+              path: token
+    extraVolumeMounts:
+      - name: token
+        mountPath: /secrets
+        readOnly: true
+
+    collector:
+      streamFields:
+        - kubernetes.pod_name
+        - kubernetes.pod_namespace
+        - kubernetes.container_name
+        - kubernetes.pod_labels.app.kubernetes.io/name
+      # Stamp the remote-cluster identity on every log line (o11y convention).
+      extraFields: '{"cluster":"${METRICS_CLUSTER_LABEL}"}'
+
+    # All staging nodes are control-plane; tolerate the taint so the DaemonSet
+    # lands everywhere.
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

kubernetes/apps/base/victoria-logs-collector/kustomization.yaml

@@ -2,5 +2,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ./deployment.yaml
-  - ./service.yaml
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml

kubernetes/apps/base/victoria-logs-collector/ocirepository.yaml

@@ -0,0 +1,14 @@
+---
+# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/source.toolkit.fluxcd.io/ocirepository_v1.json
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: victoria-logs-collector
+spec:
+  interval: 15m
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
+  ref:
+    tag: 0.3.6
+  url: oci://ghcr.io/victoriametrics/helm-charts/victoria-logs-collector

kubernetes/apps/base/vlagent/deployment.yaml

@@ -1,56 +0,0 @@
----
-# vlagent — VictoriaLogs' log-forwarding agent (the logs counterpart to vmagent).
-# Receives OTLP logs from the apps in-cluster (plaintext, no auth) and remote-
-# writes them to o11y's VictoriaLogs through vmauth, authenticating with the
-# shared bearer token. Native to the VM stack, so it speaks the same ingestion +
-# remote-write protocols as the rest of o11y.
-#
-# TODO(logs): o11y doesn't expose a VictoriaLogs ingest route on vmauth yet
-# (VLOGS_REMOTE_URL is a placeholder); wired ahead per design. Confirm the exact
-# vlagent flags + per-cluster stream field once that route lands.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: vlagent
-  labels:
-    app.kubernetes.io/name: vlagent
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: vlagent
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: vlagent
-    spec:
-      containers:
-        - name: vlagent
-          image: victoriametrics/vlagent:v1.51.0
-          args:
-            - -httpListenAddr=:9428
-            - -remoteWrite.url=${VLOGS_REMOTE_URL}
-            - -remoteWrite.bearerTokenFile=/secrets/token
-          ports:
-            - name: http
-              containerPort: 9428
-          volumeMounts:
-            - name: token
-              mountPath: /secrets
-              readOnly: true
-          readinessProbe:
-            tcpSocket: { port: http }
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 25m
-              memory: 64Mi
-            limits:
-              memory: 192Mi
-      volumes:
-        - name: token
-          secret:
-            secretName: vmagent-remote-write
-            items:
-              - key: token
-                path: token

kubernetes/apps/base/vlagent/service.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: vlagent
-  labels:
-    app.kubernetes.io/name: vlagent
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: vlagent
-  ports:
-    - name: http
-      port: 9428
-      targetPort: http

kubernetes/apps/base/vmagent/helmrelease.yaml

@@ -19,7 +19,7 @@ spec:
   values:
     # `vmagent-<name>` Service => vmagent-yucca (matches VMAGENT_OTLP). The
     # k8s-stack chart bundles the VM operator, so this one release gives us the
-    # CRDs + operator + a VMAgent. yucca-staging is a METRICS REMOTE CLUSTER:
+    # CRDs + operator + a VMAgent. yucca_staging is a METRICS REMOTE CLUSTER:
     # only the agent runs here; storage/query/alerting live at o11y.
     fullnameOverride: yucca
     global:
@@ -48,14 +48,18 @@ spec:
     vmagent:
       enabled: true
       spec:
-        # Stamp the remote-cluster identity on every series (o11y convention).
-        externalLabels:
-          cluster: ${METRICS_CLUSTER_LABEL}
+        # Stamp the remote-cluster identity on EVERY series (o11y convention).
+        # Relabel (not externalLabels): externalLabels only tags scraped data,
+        # leaving OTLP-pushed app metrics untagged. This applies to all data
+        # (scraped + ingested) before remote-write.
+        inlineRelabelConfig:
+          - target_label: cluster
+            replacement: ${METRICS_CLUSTER_LABEL}
         # Ship to o11y's vmauth (the `remote-clusters` VMUser proxies
         # /insert/0/.* -> vminsert), authenticating with the shared bearer token
-        # (TF-provisioned `vmagent-remote-write` Secret from o11y_tf_prod).
+        # (TF-provisioned `vmagent-remote-write` Secret from o11y_tf_staging).
         remoteWrite:
-          - url: https://vmauth.futostat.us/insert/0/prometheus/api/v1/write
+          - url: https://vmauth.staging.futostatus.com/insert/0/prometheus/api/v1/write
             bearerTokenSecret:
               name: vmagent-remote-write
               key: token

kubernetes/apps/base/yucca-admin-api/helmrelease.yaml

@@ -40,5 +40,3 @@ spec:
         value: info
       - name: OTEL_METRICS
         value: http://${VMAGENT_OTLP}/opentelemetry/v1/metrics
-      - name: OTEL_LOGGING
-        value: http://${VLAGENT_OTLP}/insert/opentelemetry/v1/logs

kubernetes/apps/base/yucca-api/helmrelease.yaml

@@ -56,5 +56,3 @@ spec:
         value: https://${GW_HOST}
       - name: OTEL_METRICS
         value: http://${VMAGENT_OTLP}/opentelemetry/v1/metrics
-      - name: OTEL_LOGGING
-        value: http://${VLAGENT_OTLP}/insert/opentelemetry/v1/logs

kubernetes/apps/staging/observability/kustomization.yaml

@@ -4,4 +4,4 @@ kind: Kustomization
 resources:
   - ./namespace.yaml
   - ./vmagent.yaml
-  - ./vlagent.yaml
+  - ./victoria-logs-collector.yaml

kubernetes/apps/staging/observability/namespace.yaml

@@ -3,3 +3,7 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: observability
+  # victoria-logs-collector is a DaemonSet that hostPath-mounts /var/log to tail
+  # pod logs — forbidden under Talos' default `baseline` PSS, so enforce privileged.
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kubernetes/apps/staging/observability/victoria-logs-collector.yaml

@@ -3,18 +3,18 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: vlagent
+  name: victoria-logs-collector
   namespace: flux-system
 spec:
   healthChecks:
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: vlagent
+    - apiVersion: helm.toolkit.fluxcd.io/v2
+      kind: HelmRelease
+      name: victoria-logs-collector
       namespace: observability
   interval: 1h
   retryInterval: 2m
   timeout: 5m
-  path: ./kubernetes/apps/base/vlagent
+  path: ./kubernetes/apps/base/victoria-logs-collector
   prune: true
   wait: true
   sourceRef:

kubernetes/clusters/production/cluster-settings.yaml

@@ -7,6 +7,6 @@ metadata:
   name: cluster-settings
   namespace: flux-system
 data:
-  CLUSTER_NAME: yucca-production
+  CLUSTER_NAME: yucca_production
   # TODO: real production ingress domain.
   APP_DOMAIN: yucca.futo.cloud

kubernetes/clusters/staging/cluster-settings.yaml

@@ -9,7 +9,7 @@ metadata:
   name: cluster-settings
   namespace: flux-system
 data:
-  CLUSTER_NAME: yucca-staging
+  CLUSTER_NAME: yucca_staging
   # Per-service ingress hostnames (routed by the ingress layer to the in-cluster
   # services; resolve to INGRESS_PUBLIC_IP publicly / INGRESS_INTERNAL_IP on-LAN).
   APP_DOMAIN: staging.backups.futo.cloud  # web (apex; /api routes to yucca-api)
@@ -17,7 +17,7 @@ data:
 
   OIDC_ISSUER: https://external-dev-gkhk8b.us1.zitadel.cloud
 
-  # ─── Ingress entry point (yucca-staging) ─────────────────────────────
+  # ─── Ingress entry point (yucca_staging) ─────────────────────────────
   # Internal VIP the in-cluster LB/Gateway announces (Cilium L2; distinct from
   # the 10.10.10.15 control-plane API VIP). Public IP is the NAT in front of it,
   # which staging.yucca.futo.cloud resolves to publicly. Consumed by the ingress
@@ -37,10 +37,10 @@ data:
   # Metric label stamped on everything vmagent ships (o11y convention: the
   # remote-cluster identity is the `cluster` label).
   METRICS_CLUSTER_LABEL: yucca_staging
-  # In-cluster OTLP receivers (host:port, no scheme — apps push plaintext).
-  # vmagent ingests metrics; vlagent ingests logs (both VM-native agents).
+  # In-cluster OTLP receiver for metrics (host:port, no scheme — apps push
+  # plaintext to vmagent). Logs are collected node-side by victoria-logs-collector
+  # (tails pod stdout), not pushed by the apps.
   VMAGENT_OTLP: vmagent-yucca.observability.svc:8429
-  VLAGENT_OTLP: vlagent.observability.svc:9428
-  # VictoriaLogs ingest endpoint for vlagent's egress (behind vmauth).
-  # TODO(logs): placeholder — o11y doesn't expose a logs route on vmauth yet.
-  VLOGS_REMOTE_URL: https://vmauth.futostat.us/insert/0/
+  # VictoriaLogs native ingest endpoint for the collector's egress (o11y vmauth
+  # -> vlinsert). The collector stamps the cluster label via collector.extraFields.
+  VLOGS_REMOTE_URL: https://vmauth.staging.futostatus.com/insert/native

tf/.env

@@ -25,8 +25,8 @@ export TF_VAR_yucca_oidc_client_secret="op://yucca_tf_staging/CUSTOMER_ZITADEL_O
 export TF_VAR_yucca_rgw_access_key_id="op://yucca_tf_staging/SIETCH_CEPH_S3_SVC_YUCCA_RESTIC_ACCESS_KEY/password"
 export TF_VAR_yucca_rgw_secret_access_key="op://yucca_tf_staging/SIETCH_CEPH_S3_SVC_YUCCA_RESTIC_SECRET_KEY/password"
 
-# vmagent + vlagent → o11y vmauth bearer token.
-export TF_VAR_vmauth_remote_write_password="op://yucca_tf_staging/VICTORIAMETRICS_VMAUTH_PASSWORD/password"
+# vmagent + collector → o11y staging vmauth bearer token.
+export TF_VAR_vmauth_remote_write_password="op://o11y_tf_staging/VICTORIAMETRICS_VMAUTH_PASSWORD/password"
 
 # Cloudflare API token for cert-manager DNS-01 (same item, TF_VAR form).
 export TF_VAR_cloudflare_api_token="op://yucca_tf_staging/CLOUDFLARE_API_TOKEN/password"

tf/deployment/staging/talos/secrets.tf

@@ -8,7 +8,7 @@
 #                    with the public one — they MUST be the same pair).
 #   • OIDC / RGW   — externally issued, human-managed in 1P; read via TF_VAR
 #                    (op:// refs in tf/.env) and written into the app Secrets.
-#   • vmauth token — shared o11y credential (o11y_tf_prod), for vmagent egress.
+#   • vmauth token — shared o11y credential (o11y_tf_staging), for vmagent egress.
 #
 # Each Secret is named after its chart's fullnameOverride so the chart's own
 # `secretData` fixture (nulled in the staging HelmRelease) cedes the name and

tf/deployment/staging/talos/variables.tf

@@ -32,7 +32,7 @@ variable "flux_github_app_private_key" {
 # ─── App secrets (secrets.tf) ───────────────────────────────────────────
 #
 # Externally-issued / human-managed secrets. Live in 1P (yucca_tf_staging_manual
-# for app creds, o11y_tf_prod for the shared vmauth token) and are injected via
+# for app creds, o11y_tf_staging for the shared vmauth token) and are injected via
 # TF_VAR from op:// refs in tf/.env. Empty defaults keep `tofu validate` clean
 # and let the staging slice deploy before the real values are populated — the
 # apps come up, just without working OIDC / object storage / metrics egress.
@@ -87,10 +87,10 @@ variable "yucca_rgw_secret_access_key" {
 }
 
 # Bearer token vmagent uses to remote-write metrics to o11y's vmauth. This is
-# the shared VICTORIAMETRICS_VMAUTH_PASSWORD from the o11y_tf_prod vault (the
+# the shared VICTORIAMETRICS_VMAUTH_PASSWORD from the o11y_tf_staging vault (the
 # `remote-clusters` VMUser authenticates remote clusters with it).
 variable "vmauth_remote_write_password" {
-  description = "o11y vmauth bearer token for vmagent remote-write. Injected via TF_VAR from 1P (o11y_tf_prod/VICTORIAMETRICS_VMAUTH_PASSWORD)."
+  description = "o11y vmauth bearer token for vmagent remote-write. Injected via TF_VAR from 1P (o11y_tf_staging/VICTORIAMETRICS_VMAUTH_PASSWORD)."
   type        = string
   sensitive   = true
   default     = ""