mirror of
https://github.com/immich-app/devtools.git
synced 2026-09-30 13:23:00 +08:00
feat(data): recording rules for data.immich.app charts (#1932)
This commit is contained in:
bo0tzz
@@ -55,7 +55,23 @@ spec:
|
||||
alertmanager:
|
||||
enabled: false
|
||||
vmalert:
|
||||
enabled: false
|
||||
enabled: true
|
||||
spec:
|
||||
# Only evaluate our own recording rules (vmrule-data-pipeline.yaml).
|
||||
selectAllByDefault: false
|
||||
ruleSelector:
|
||||
matchLabels:
|
||||
immich.app/vmrule: data-pipeline
|
||||
extraArgs:
|
||||
# No alertmanager is deployed; vmalert only runs recording rules.
|
||||
notifier.blackhole: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
grafana:
|
||||
enabled: false
|
||||
kubeEtcd:
|
||||
|
||||
@@ -3,3 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- helmrelease.yaml
|
||||
- vmrule-data-pipeline.yaml
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
---
|
||||
# Recording rules for the data.immich.app charts.
|
||||
#
|
||||
# The raw immich_data_repository_* GitHub metrics carry the event sender as
|
||||
# labels (username, user_id), so e.g. star_total has one series per stargazer
|
||||
# (>100k). The site only ever needs the repo-wide maximum, and scanning every
|
||||
# series for the full history from 2022 takes 20-30s cold and times out. These
|
||||
# rules drop the per-user labels once per hour so the API reads a single series
|
||||
# per repository instead. Keep replay/configmap-rules.yaml in sync.
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: data-pipeline-recording
|
||||
namespace: data
|
||||
labels:
|
||||
immich.app/vmrule: data-pipeline
|
||||
spec:
|
||||
groups:
|
||||
- name: data-pipeline.repository
|
||||
interval: 1h
|
||||
rules:
|
||||
- record: immich_data:repository_star_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_star_total[1h]))
|
||||
- record: immich_data:repository_issue_open_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_issue_open_total[1h]))
|
||||
- record: immich_data:repository_pull_request_open_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_pull_request_open_total[1h]))
|
||||
- record: immich_data:repository_pull_request_merged_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_pull_request_merged_total[1h]))
|
||||
- record: immich_data:repository_discussion_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_discussion_total[1h]))
|
||||
- record: immich_data:repository_reddit_subscriber_total:max1h
|
||||
expr: |-
|
||||
max by (environment) (max_over_time(immich_data_repository_reddit_subscriber_total[1h]))
|
||||
- record: immich_data:repository_discord_member_total:max1h
|
||||
expr: |-
|
||||
max by (environment) (max_over_time(immich_data_repository_discord_member_total[1h]))
|
||||
@@ -42,6 +42,29 @@ spec:
|
||||
---
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: data-pipeline-vmetrics-replay
|
||||
namespace: flux-system
|
||||
spec:
|
||||
commonMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: data-pipeline-vmetrics-replay
|
||||
dependsOn:
|
||||
- name: data-pipeline-vmetrics
|
||||
path: ./kubernetes/apps/pipelines/data/victoria-metrics/replay
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: immich-kubernetes
|
||||
# wait deliberately off: the replay Job can run for a while; flux should
|
||||
# apply it and move on rather than flap on health-check timeouts.
|
||||
wait: false
|
||||
interval: 30m
|
||||
retryInterval: 1m
|
||||
timeout: 5m
|
||||
---
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: data-pipeline-vmetrics-backup-secrets
|
||||
namespace: flux-system
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# vmalert replay — data pipeline recording rules backfill
|
||||
|
||||
One-off backfill of history for the recording rules defined in
|
||||
`../app/vmrule-data-pipeline.yaml`. Flux applies this directory once via the
|
||||
`data-pipeline-vmetrics-replay` Kustomization in `../ks.yaml` (after
|
||||
`data-pipeline-vmetrics`, so vmsingle, vmalert, and the VMRule are already in
|
||||
place). No manual steps are needed on merge.
|
||||
|
||||
The Job replays the rule group from `-replay.timeFrom` (Feb 2022, before the
|
||||
first ingested sample) to now, then resets the vmsingle rollup result cache so
|
||||
queries immediately see the backfilled samples. The data vmsingle keeps 50y of
|
||||
raw data, so the whole history is covered.
|
||||
|
||||
## Monitoring
|
||||
|
||||
```sh
|
||||
kubectl -n data logs -f job/data-pipeline-replay -c replay
|
||||
kubectl -n data wait --for=condition=complete --timeout=1h job/data-pipeline-replay
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
Compare a rule series against the raw query the site used before, e.g. in the
|
||||
data Grafana or with `curl` against vmsingle:
|
||||
|
||||
```
|
||||
interpolate(max(max_over_time(immich_data:repository_star_total:max1h{environment="prod",repository_name="immich"}[1d])))
|
||||
interpolate(max(max_over_time(immich_data_repository_star_total{environment="prod",repository_name="immich"}[1d])))
|
||||
```
|
||||
|
||||
Both should produce the same daily values. The second one is the expensive
|
||||
scan; expect it to take 20-30s.
|
||||
|
||||
## Cleanup
|
||||
|
||||
After the job completes and the rule series validate: remove this directory
|
||||
and the `data-pipeline-vmetrics-replay` Kustomization from `../ks.yaml` in a
|
||||
follow-up PR — `prune: true` then deletes the Job and ConfigMap.
|
||||
|
||||
## Re-running
|
||||
|
||||
Jobs are immutable, so the Job carries the `kustomize.toolkit.fluxcd.io/force`
|
||||
annotation: flux will delete+recreate (and therefore re-run the replay) on any
|
||||
spec change. Manually deleting the completed Job also makes flux recreate it on
|
||||
the next reconciliation — a re-replay overwrites the same historical points
|
||||
with identical values, so this is harmless.
|
||||
|
||||
## Notes
|
||||
|
||||
- If vmalert is down for a period later on, the rule series will have holes
|
||||
for that period unless the affected window is re-replayed.
|
||||
- `configmap-rules.yaml` must be kept in sync with
|
||||
`../app/vmrule-data-pipeline.yaml` whenever the rules change — the replay
|
||||
file is a plain vmalert rule-file copy of the VMRule group.
|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
# Rule file for the one-off vmalert replay (backfill of recording rule history).
|
||||
# Applied once by flux (ks.yaml: data-pipeline-vmetrics-replay) together with job.yaml; see README.md.
|
||||
# MUST be kept in sync with app/vmrule-data-pipeline.yaml if rules change.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: data-pipeline-replay-rules
|
||||
namespace: data
|
||||
data:
|
||||
rules.yaml: |
|
||||
groups:
|
||||
- name: data-pipeline.repository
|
||||
interval: 1h
|
||||
rules:
|
||||
- record: immich_data:repository_star_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_star_total[1h]))
|
||||
- record: immich_data:repository_issue_open_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_issue_open_total[1h]))
|
||||
- record: immich_data:repository_pull_request_open_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_pull_request_open_total[1h]))
|
||||
- record: immich_data:repository_pull_request_merged_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_pull_request_merged_total[1h]))
|
||||
- record: immich_data:repository_discussion_total:max1h
|
||||
expr: |-
|
||||
max by (environment, org_name, repository_name) (max_over_time(immich_data_repository_discussion_total[1h]))
|
||||
- record: immich_data:repository_reddit_subscriber_total:max1h
|
||||
expr: |-
|
||||
max by (environment) (max_over_time(immich_data_repository_reddit_subscriber_total[1h]))
|
||||
- record: immich_data:repository_discord_member_total:max1h
|
||||
expr: |-
|
||||
max by (environment) (max_over_time(immich_data_repository_discord_member_total[1h]))
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
# One-off vmalert replay Job to backfill recording rule history, applied once
|
||||
# by flux (ks.yaml: data-pipeline-vmetrics-replay). Once it has completed and
|
||||
# the rule series validate against the raw queries, remove this directory and
|
||||
# the data-pipeline-vmetrics-replay Kustomization in a follow-up PR — prune
|
||||
# will delete the Job and ConfigMap.
|
||||
#
|
||||
# The data vmsingle keeps 50y of raw samples, so unlike the monitoring replay
|
||||
# this one covers the full history from the first ingested sample (Feb 2022).
|
||||
# Once the rules are replayed the vmsingle rollup result cache is reset so
|
||||
# queries immediately see the backfilled samples.
|
||||
#
|
||||
# Jobs are immutable; the force annotation makes flux delete+recreate (and
|
||||
# therefore re-run the replay) if this spec is ever changed.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: data-pipeline-replay
|
||||
namespace: data
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/force: enabled
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: data-pipeline-replay
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: replay
|
||||
# Pinned to chart 0.91.2's vmalert version (Chart.yaml appVersion v1.150.0).
|
||||
image: victoriametrics/vmalert:v1.150.0@sha256:c6e6c1ef6e43c09510dd0aff264bf0ea319c1bdfced1ccc79dad1545950a7989
|
||||
args:
|
||||
- -rule=/rules/rules.yaml
|
||||
- -datasource.url=http://vmsingle-vmetrics-data:8428
|
||||
- -remoteWrite.url=http://vmsingle-vmetrics-data:8428
|
||||
- -replay.timeFrom=2022-02-01T00:00:00Z
|
||||
- -replay.disableProgressBar=true
|
||||
# 100 hourly points per query_range = ~4 days of raw data per
|
||||
# request; measured at ~0.15s each against the live instance.
|
||||
- -replay.maxDatapointsPerQuery=100
|
||||
# vmsingle runs with -search.maxConcurrentRequests=2, so replay
|
||||
# queries can be rejected while the site is being queried.
|
||||
- -replay.ruleRetryAttempts=10
|
||||
volumeMounts:
|
||||
- name: rules
|
||||
mountPath: /rules
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: reset-rollup-cache
|
||||
image: curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13
|
||||
args:
|
||||
- -sf
|
||||
- http://vmsingle-vmetrics-data:8428/internal/resetRollupResultCache
|
||||
volumes:
|
||||
- name: rules
|
||||
configMap:
|
||||
name: data-pipeline-replay-rules
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./configmap-rules.yaml
|
||||
- ./job.yaml
|
||||
Reference in New Issue
Block a user