feat: full flux manifests for o11y stack (#4)

2026-09-30 13:23:23 +08:00 · 2026-02-23 13:50:48 +00:00
parent 5d80d30e86
commit c971baa727
33 changed files with 673 additions and 0 deletions
@@ -0,0 +1,22 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cert-manager
+  namespace: network
+spec:
+  chartRef:
+    kind: OCIRepository
+    name: cert-manager
+  interval: 1h
+  values:
+    crds:
+      enabled: true
+
+    resources:
+      requests:
+        cpu: 10m
+        memory: 64Mi
+      limits:
+        cpu: 200m
+        memory: 256Mi
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: cert-manager
+  namespace: network
+spec:
+  interval: 15m
+  url: oci://quay.io/jetstack/charts/cert-manager
+  ref:
+    tag: v1.19.3
+
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  targetNamespace: network
+  interval: 1h
+  path: ./kubernetes/apps/infra/cert-manager/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./cert-manager/ks.yaml
+  - ./victoria-metrics-operator-crds/ks.yaml
+  - ./victoria-metrics-operator/ks.yaml
+  - ./openebs/ks.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: storage
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: operators
@@ -0,0 +1,49 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: openebs
+  namespace: storage
+spec:
+  chartRef:
+    kind: OCIRepository
+    name: openebs
+  interval: 1h
+  values:
+    localpv-provisioner:
+      localpv:
+        basePath: &hostPath /var/mnt/u-hostpath
+        replicas: 1
+        enableLeaderElection: true
+        requests:
+          cpu: 20m
+          memory: 64Mi
+        limits:
+          memory: 128Mi
+
+      hostpathClass:
+        enabled: true
+        name: openebs-hostpath
+        reclaimPolicy: Delete
+        isDefaultClass: true
+        basePath: *hostPath
+
+    analytics:
+      enabled: false
+    alloy:
+      enabled: false
+    loki:
+      enabled: false
+    engines:
+      local:
+        lvm:
+          enabled: false
+        zfs:
+          enabled: false
+      replicated:
+        mayastor:
+          enabled: false
+    openebs-crds:
+      csi:
+        volumeSnapshots:
+          enabled: false
+          keep: false
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: cert-manager
+  namespace: storage
+spec:
+  interval: 15m
+  url: oci://ghcr.io/home-operations/charts-mirror/openebs
+  ref:
+    tag: 4.4.0
+
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openebs
+  namespace: flux-system
+spec:
+  targetNamespace: storage
+  interval: 1h
+  path: ./kubernetes/apps/infra/openebs/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
@@ -0,0 +1,12 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: victoria-metrics-operator-crds
+  namespace: operators
+spec:
+  chartRef:
+    kind: OCIRepository
+    name: victoria-metrics-operator-crds
+  interval: 1h
+  values: {}
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: victoria-metrics-operator-crds
+  namespace: operators
+spec:
+  interval: 15m
+  url: oci://ghcr.io/victoriametrics/helm-charts/victoria-metrics-operator-crds
+  ref:
+    tag: 0.6.1
+
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: victoria-metrics-operator-crds
+  namespace: flux-system
+spec:
+  targetNamespace: operators
+  interval: 1h
+  path: ./kubernetes/apps/infra/victoria-metrics-operator-crds/app
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
@@ -0,0 +1,28 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: victoria-metrics-operator
+  namespace: operators
+spec:
+  chartRef:
+    kind: OCIRepository
+    name: victoria-metrics-operator
+  interval: 1h
+  values:
+    admissionWebhooks:
+      enabled: true
+      certManager:
+        enabled: true
+
+    operator:
+      prometheus_converter:
+        enabled: true
+
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 500m
+        memory: 512Mi
@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./issuer.yaml
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: victoria-metrics-operator
+  namespace: operators
+spec:
+  interval: 15m
+  url: oci://ghcr.io/victoriametrics/helm-charts/victoria-metrics-operator
+  ref:
+    tag: 0.58.1
+
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: victoria-metrics-operator
+  namespace: flux-system
+spec:
+  targetNamespace: operators
+  dependsOn:
+    - name: cert-manager
+    - name: victoria-metrics-operator-crds
+  interval: 1h
+  path: ./kubernetes/apps/infra/victoria-metrics-operator/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./infra/
+  - ./o11y/
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: o11y
@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./victoria-metrics-stack/ks.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./vmsingle.yaml
+  - ./vmagent.yaml
+  - ./vmauth-external.yaml
+  - ./vmauth-internal.yaml
+  - ./vmusers/
@@ -0,0 +1,65 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAgent
+metadata:
+  name: vmagent
+  namespace: o11y
+spec:
+  replicaCount: 2
+
+  remoteWrite:
+    - url: "http://vmauth-internal:8427/api/v1/write" # TODO: Check the actual svc address lol
+      basicAuth:
+        username:
+          name: vmagent-credentials
+          key: username
+        password:
+          name: vmagent-credentials
+          key: password
+
+    - url: "http://${ZONE_2_NODE_IP}:30426/api/v1/write"
+      basicAuth:
+        username:
+          name: vmagent-credentials
+          key: username
+        password:
+          name: vmagent-credentials
+          key: password
+
+    - url: "http://${ZONE_3_NODE_IP}:30426/api/v1/write"
+      basicAuth:
+        username:
+          name: vmagent-credentials
+          key: username
+        password:
+          name: vmagent-credentials
+          key: password
+
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+
+  serviceSpec:
+    type: ClusterIP
+    spec:
+      ports:
+        - name: http
+          port: 8429
+          targetPort: 8429
+
+  persistentVolume:
+    storageClassName: openebs-hostpath
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 50Gi
+
+  extraArgs:
+    remoteWrite.maxDiskUsagePerURL: "10GB"
+    remoteWrite.queues: "3"
+    memory.allowedPercent: "80"
@@ -0,0 +1,32 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAuth
+metadata:
+  name: vmauth-external
+  namespace: o11y
+spec:
+  replicaCount: 2
+
+  userSelector:
+    matchLabels:
+      type: external
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+  serviceSpec:
+    type: NodePort
+    spec:
+      ports:
+        - name: http
+          port: 8427
+          targetPort: 8427
+          nodePort: 30427
+
+  extraArgs:
+    maxIdleConnsPerBackend: "100"
@@ -0,0 +1,32 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAuth
+metadata:
+  name: vmauth-internal
+  namespace: o11y
+spec:
+  replicaCount: 2
+
+  userSelector:
+    matchLabels:
+      type: internal
+
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 250m
+      memory: 256Mi
+
+  serviceSpec:
+    type: NodePort
+    spec:
+      ports:
+        - name: http
+          port: 8427
+          targetPort: 8427
+          nodePort: 30426
+
+  extraArgs:
+    maxIdleConnsPerBackend: "100"
@@ -0,0 +1,39 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMSingle
+metadata:
+  name: vmsingle
+  namespace: o11y
+spec:
+  replicaCount: 1
+  retentionPeriod: "60d"
+
+  storage:
+    storageClassName: openebs-hostpath
+    accessModes:
+      - ReadWriteOnce
+    resources:
+      requests:
+        storage: 300Gi
+
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 4Gi
+    limits:
+      cpu: 4000m
+      memory: 16Gi
+
+  serviceSpec:
+    type: ClusterIP
+    spec:
+      ports:
+        - name: http
+          port: 8428
+          targetPort: 8428
+
+  extraArgs:
+    dedup.minScrapeInterval: "30s"
+    search.maxQueryDuration: "5m"
+    search.maxConcurrentRequests: "16"
+    memory.allowedPercent: "60"
@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./vmuser-external-write.yaml
+  - ./vmuser-external-read.yaml
+  - ./vmuser-internal-write.yaml
+  - ./vmuser-internal-read.yaml
@@ -0,0 +1,47 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMUser
+metadata:
+  name: vmuser-external-read
+  namespace: o11y
+  labels:
+    type: external
+spec:
+  username: "reader"
+  passwordRef:
+    name: vmauth-external-credentials
+    key: reader-password
+
+  targetRefs:
+    - static:
+        url: "http://${ZONE_1_NODE_IP}:30426"
+      paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/series"
+        - "/api/v1/labels"
+        - "/api/v1/label/.*/values"
+        - "/prometheus/api/v1/.*"
+
+    - static:
+        url: "http://${ZONE_2_NODE_IP}:30426"
+      paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/series"
+        - "/api/v1/labels"
+        - "/api/v1/label/.*/values"
+        - "/prometheus/api/v1/.*"
+
+    - static:
+        url: "http://${ZONE_3_NODE_IP}:30426"
+      paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/series"
+        - "/api/v1/labels"
+        - "/api/v1/label/.*/values"
+        - "/prometheus/api/v1/.*"
+
+  load_balancing_policy: "first_available"
+  retry_status_codes: [429, 500, 502, 503, 504]
@@ -0,0 +1,26 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMUser
+metadata:
+  name: vmuser-external-write
+  namespace: o11y
+  labels:
+    type: external
+spec:
+  username: "writer"
+  passwordRef:
+    name: vmauth-external-credentials
+    key: writer-password
+
+  targetRefs:
+    - crd:
+        kind: VMAgent
+        name: vmagent
+        namespace: o11y
+      target_path_suffix: "/api/v1/write"
+      paths:
+        - "/api/v1/write"
+        - "/prometheus/api/v1/write"
+
+  retry_status_codes: [429, 500, 502, 503, 504]
+  max_concurrent_requests: 1000
@@ -0,0 +1,28 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMUser
+metadata:
+  name: vmuser-internal-read
+  namespace: o11y
+  labels:
+    type: internal
+spec:
+  username: "vmauth-reader"
+  passwordRef:
+    name: vmauth-internal-credentials
+    key: vmauth-reader-password
+
+  targetRefs:
+    - crd:
+        kind: VMSingle
+        name: vmsingle
+        namespace: o11y
+      paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/series"
+        - "/api/v1/labels"
+        - "/api/v1/label/.*/values"
+        - "/prometheus/api/v1/.*"
+
+  retry_status_codes: [429, 500, 502, 503, 504]
@@ -0,0 +1,24 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMUser
+metadata:
+  name: vmuser-internal-write
+  namespace: o11y
+  labels:
+    type: internal
+spec:
+  username: "vmagent"
+  passwordRef:
+    name: vmauth-internal-credentials
+    key: vmagent-password
+
+  targetRefs:
+    - crd:
+        kind: VMSingle
+        name: vmsingle
+        namespace: o11y
+      target_path_suffix: "/api/v1/write"
+      paths:
+        - "/api/v1/write"
+
+  retry_status_codes: [429, 500, 502, 503, 504]
@@ -0,0 +1,23 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: victoria-metrics-stack
+  namespace: flux-system
+spec:
+  targetNamespace: o11y
+  dependsOn:
+    - name: victoria-metrics-operator
+    - name: openebs
+  interval: 1h
+  path: ./kubernetes/apps/o11y/victoria-metrics-stack/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: vm-zone-endpoints
+        optional: false
@@ -0,0 +1,41 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps
+  namespace: flux-system
+spec:
+  interval: 1h
+  path: ./kubernetes/apps
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: false
+  patches:
+    - patch: |-
+        apiVersion: kustomize.toolkit.fluxcd.io/v1
+        kind: Kustomization
+        metadata:
+          name: _
+        spec:
+          patches:
+            - patch: |-
+                apiVersion: helm.toolkit.fluxcd.io/v2
+                kind: HelmRelease
+                metadata:
+                  name: _
+                spec:
+                  install:
+                    crds: CreateReplace
+                  upgrade:
+                    cleanupOnFail: true
+                    crds: CreateReplace
+                    remediation:
+                      retries: 2
+              target:
+                group: helm.toolkit.fluxcd.io
+                kind: HelmRelease
+      target:
+        group: kustomize.toolkit.fluxcd.io
+        kind: Kustomization