feat: metrics worker (radosgw ingest) (#123)

This commit is contained in:
Paul Makles
2026-06-17 14:16:28 +01:00
committed by GitHub
parent db6415c6fa
commit faa288087d
57 changed files with 1230 additions and 34 deletions
+2 -2
View File
@@ -9,8 +9,8 @@
# rebuilds charts/*/charts/ (rm -rf + dependency build) and races the renders.
set -euo pipefail
LIB_CONSUMERS=(yucca-api yucca-admin-api web michael mock-oidc)
ALL_CHARTS=(yucca-api yucca-admin-api web michael mock-oidc cnpg-cluster ceph-objectuser rook-ceph-cluster)
LIB_CONSUMERS=(yucca-api yucca-admin-api yucca-metrics-worker web michael mock-oidc)
ALL_CHARTS=(yucca-api yucca-admin-api yucca-metrics-worker web michael mock-oidc cnpg-cluster ceph-objectuser rook-ceph-cluster)
echo "==> helm dependency build (yucca-common consumers)"
for c in "${LIB_CONSUMERS[@]}"; do
+9
View File
@@ -55,6 +55,15 @@ S3_REGION=us-east-1
S3_FORCE_PATH_STYLE=true
export S3_ENDPOINT S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY S3_REGION S3_FORCE_PATH_STYLE
if kubectl -n yucca get secret rook-ceph-object-user-yucca-metrics >/dev/null 2>&1; then
RADOS_ENDPOINT=http://localhost:9000
RADOS_ACCESS_KEY_ID="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-metrics -o jsonpath='{.data.AccessKey}' | base64 -d)"
RADOS_SECRET_ACCESS_KEY="$(kubectl -n yucca get secret rook-ceph-object-user-yucca-metrics -o jsonpath='{.data.SecretKey}' | base64 -d)"
export RADOS_ENDPOINT RADOS_ACCESS_KEY_ID RADOS_SECRET_ACCESS_KEY
else
echo "==> rook-ceph-object-user-yucca-metrics not found; RGW integration tests will skip"
fi
# The deployed mock-oidc advertises its in-cluster name as the issuer; the dns
# preload resolves it to the port-forward for every node process (jest + the
# apps it boots). Same split-horizon glue as the e2e runner.
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
#MISE description="Build yucca-metrics-worker"
#MISE depends=["common:build"]
set -e
NODE_ENV=production pnpm --filter yucca-metrics-worker build
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
#MISE description="Run yucca-metrics-worker in development mode"
#MISE depends=["common:build"]
#MISE dir="{{config_root}}"
set -e
source "$(dirname "$0")/env"
if command -v op &>/dev/null; then
op run --env-file="./.env" -- pnpm --filter yucca-metrics-worker start:dev
else
pnpm --filter yucca-metrics-worker start:dev
fi
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
export YUCCA_METRICS_WORKER_PORT=${YUCCA_METRICS_WORKER_PORT:-3040}
export POSTGRES_HOST=${POSTGRES_HOST:-localhost}
export POSTGRES_USERNAME=${POSTGRES_USERNAME:-postgres}
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
export POSTGRES_DATABASE=${POSTGRES_DATABASE:-yucca}
export POSTGRES_PORT=${POSTGRES_PORT:-15432}
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
#MISE description="Run integration tests for yucca-metrics-worker"
set -e
source "$(dirname "$0")/../env"
if [[ -z "${RADOS_ENDPOINT:-}" || -z "${RADOS_ACCESS_KEY_ID:-}" || -z "${RADOS_SECRET_ACCESS_KEY:-}" ]]; then
echo "RADOS_* not set; skipping yucca-metrics-worker integration tests"
exit 0
fi
pnpm --filter yucca-metrics-worker test:integration "$@"
+42 -3
View File
@@ -175,6 +175,34 @@ docker_build(
],
)
docker_build(
'yucca-metrics-worker',
context='.',
dockerfile='packages/yucca-metrics-worker/Dockerfile',
target='dev',
only=[
'./pnpm-workspace.yaml',
'./pnpm-lock.yaml',
'./package.json',
'./.npmrc',
'./packages',
],
ignore=[
'**/node_modules',
'**/dist',
'**/.svelte-kit',
'packages/michael',
'packages/e2e',
],
live_update=[
sync('./packages/yucca-metrics-worker', '/app/packages/yucca-metrics-worker'),
sync('./packages/common', '/app/packages/common'),
# yucca-metrics-worker/src/schema is a symlink into yucca-api; keep it synced.
sync('./packages/yucca-api', '/app/packages/yucca-api'),
run('cd /app && pnpm --filter @common/server build', trigger=['./packages/common/src']),
],
)
# mock-oidc-provider has no dev target (config-only via env); a plain build is
# enough — it rarely changes and is reconfigured through Helm values.
docker_build(
@@ -200,10 +228,11 @@ docker_build(
# ---------------------------------------------------------------------------
local_resource(
'helm-deps',
cmd='rm -rf charts/yucca-api/charts charts/yucca-admin-api/charts charts/web/charts charts/michael/charts charts/mock-oidc/charts && for d in charts/yucca-api charts/yucca-admin-api charts/web charts/michael charts/mock-oidc; do (cd $d && helm dependency build); done',
cmd='rm -rf charts/yucca-api/charts charts/yucca-admin-api/charts charts/yucca-metrics-worker/charts charts/web/charts charts/michael/charts charts/mock-oidc/charts && for d in charts/yucca-api charts/yucca-admin-api charts/yucca-metrics-worker charts/web charts/michael charts/mock-oidc; do (cd $d && helm dependency build); done',
deps=[
'charts/yucca-api',
'charts/yucca-admin-api',
'charts/yucca-metrics-worker',
'charts/web',
'charts/michael',
'charts/mock-oidc',
@@ -231,6 +260,7 @@ APP_WIRING = {
# dev_env: receives the .env override Secret (see load_dev_env above).
'yucca-api': {'build': 'yucca-api', 'deps': ['yucca-database', 'yucca-mock-oidc', 'yucca-michael'], 'dev_env': True},
'yucca-admin-api': {'build': 'yucca-admin-api', 'deps': ['yucca-database', 'yucca-mock-oidc']},
'yucca-metrics-worker': {'build': 'yucca-metrics-worker', 'deps': ['yucca-database', 'yucca-metrics-object-user'], 'dev_env': True},
'yucca-web': {'build': 'web', 'deps': ['yucca-api']},
'yucca-michael': {'build': 'michael', 'deps': ['yucca-object-user']},
'yucca-mock-oidc': {'build': 'mock-oidc-provider', 'deps': []},
@@ -239,6 +269,10 @@ APP_WIRING = {
# RGW user), so Tilt's pod tracking would hang at "pending". Mark ready on
# apply; michael still waits on this resource for ordering.
'yucca-object-user': {'build': None, 'deps': ['rook-ceph-cluster'], 'pod_readiness': 'ignore'},
# Shares charts/ceph-objectuser with yucca-object-user; its userName/caps
# come from the HelmRelease values, so dev must apply them (dev_values) or
# both releases would default to userName=michael and collide.
'yucca-metrics-object-user': {'build': None, 'deps': ['rook-ceph-cluster'], 'pod_readiness': 'ignore', 'dev_values': True},
# Rook spins up transient mon/osd "canary" pods and deletes them; Tilt's
# pod tracking misreads those deletions as failures. Ignore pod readiness
# here — real convergence is still gated downstream (object-user -> michael
@@ -278,8 +312,10 @@ def discover_apps():
namespace = hr['metadata'].get('namespace', 'yucca')
if source.get('kind') == 'GitRepository' and chart.startswith('charts/'):
# In-repo chart: dev renders it with its values.yaml defaults; the
# HelmRelease's .spec.values are the prod-side overrides.
local_apps.append(struct(name=name, namespace=namespace, chart=chart))
# HelmRelease's .spec.values are the prod-side overrides. Apps that
# opt in via dev_values (two releases sharing one chart, where the
# defaults aren't enough to tell them apart) get them in dev too.
local_apps.append(struct(name=name, namespace=namespace, chart=chart, values=hr['spec'].get('values', {})))
elif source.get('kind') == 'HelmRepository':
# Remote chart: dev installs the exact version + values Flux would.
remote_apps.append(struct(
@@ -327,6 +363,9 @@ for app in LOCAL_APPS:
builds = [wiring['build']] if wiring['build'] else []
flags = ['--timeout=10m']
extra_deps = []
if wiring.get('dev_values'):
for key in sorted(app.values.keys()):
flags += ['--set-json', '%s=%s' % (key, str(encode_json(app.values[key])).rstrip('\n'))]
if DEV_ENV and wiring.get('dev_env'):
flags += ['--set-json', 'extraEnvFrom=[{"secretRef":{"name":"yucca-dev-env"}}]']
for key, value_path in DEV_ENV_VALUE_KEYS.items():
@@ -6,3 +6,7 @@ spec:
store: {{ .Values.store }}
clusterNamespace: {{ .Values.clusterNamespace }}
displayName: {{ .Values.userName }}
{{- with .Values.capabilities }}
capabilities:
{{- toYaml . | nindent 4 }}
{{- end }}
+6
View File
@@ -5,3 +5,9 @@ userName: michael
# CephObjectStore name + the namespace where the Rook cluster/objectstore lives.
store: yucca
clusterNamespace: rook-ceph
# RGW admin capabilities (radosgw-admin caps). Empty = a plain S3 user that can
# only manage its own buckets (michael's case). Grant read caps to a user that
# needs the RGW admin API — e.g. the metrics worker reading per-bucket usage.
# See https://rook.io/docs/rook/latest/CRDs/Object-Storage/ceph-object-store-user-crd/
capabilities: {}
+4 -8
View File
@@ -37,14 +37,10 @@ env:
value: debug
- name: OIDC_ADMIN_ALLOW_INSECURE
value: "true"
- name: OTLP_METRICS_ENDPOINT
value: victoria-metrics:8428
- name: OTLP_METRICS_URL_PATH
value: /opentelemetry/v1/metrics
- name: OTLP_LOGS_ENDPOINT
value: victoria-logs:9428
- name: OTLP_LOGS_URL_PATH
value: /insert/opentelemetry/v1/logs
- name: OTEL_METRICS
value: http://victoria-metrics:8428/opentelemetry/v1/metrics
- name: OTEL_LOGGING
value: http://victoria-logs:9428/insert/opentelemetry/v1/logs
# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process
# still counts as Ready (this masked two real bugs). The startupProbe budgets
+4 -8
View File
@@ -69,14 +69,10 @@ env:
value: yucca-michael
- name: RESTIC_API_PORT
value: "3010"
- name: OTLP_METRICS_ENDPOINT
value: victoria-metrics:8428
- name: OTLP_METRICS_URL_PATH
value: /opentelemetry/v1/metrics
- name: OTLP_LOGS_ENDPOINT
value: victoria-logs:9428
- name: OTLP_LOGS_URL_PATH
value: /insert/opentelemetry/v1/logs
- name: OTEL_METRICS
value: http://victoria-metrics:8428/opentelemetry/v1/metrics
- name: OTEL_LOGGING
value: http://victoria-logs:9428/insert/opentelemetry/v1/logs
# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process
# still counts as Ready (this masked two real bugs). The startupProbe budgets
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v2
name: yucca-metrics-worker
description: Yucca metrics worker (NestJS)
type: application
version: 0.1.0
appVersion: "0.0.1"
dependencies:
- name: yucca-common
version: 0.1.0
repository: "file://../yucca-common"
@@ -0,0 +1,16 @@
{{- $_ := set .Values "env" (concat .Values.env (list
(dict "name" "RADOS_ENDPOINT" "value" .Values.radosEndpoint)
(dict "name" "RADOS_ACCESS_KEY_ID" "valueFrom" (dict "secretKeyRef" (dict "name" .Values.radosSecretName "key" "AccessKey")))
(dict "name" "RADOS_SECRET_ACCESS_KEY" "valueFrom" (dict "secretKeyRef" (dict "name" .Values.radosSecretName "key" "SecretKey")))
(dict "name" "POSTGRES_HOST" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "host")))
(dict "name" "POSTGRES_PORT" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "port")))
(dict "name" "POSTGRES_DATABASE" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "dbname")))
(dict "name" "POSTGRES_USERNAME" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "username")))
(dict "name" "POSTGRES_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict "name" (printf "%s-app" .Values.postgresClusterName) "key" "password")))
)) }}
{{- /* extraEnvFrom comes after the chart secret: for duplicate keys Kubernetes
takes the LAST envFrom source, so these act as overrides. */}}
{{- $_ := set .Values "envFrom" (concat
(list (dict "secretRef" (dict "name" (include "yucca-common.fullname" .))))
(.Values.extraEnvFrom | default (list))) }}
{{- include "yucca-common.deployment" . }}
@@ -0,0 +1 @@
{{- include "yucca-common.secret" . }}
+56
View File
@@ -0,0 +1,56 @@
replicas: 1
# Stable in-cluster name, independent of the Helm release name (dev == prod).
fullnameOverride: yucca-metrics-worker
image:
repository: k3d-registry.localhost:5000/yucca-metrics-worker
tag: dev
pullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3040
# CNPG-managed postgres Cluster name (shares yucca-api's database)
postgresClusterName: yucca-db
# Rook-Ceph RGW endpoint + the object-user Secret holding AccessKey/SecretKey.
# A dedicated RGW user (charts/ceph-objectuser via the yucca-metrics-object-user
# release) with admin read caps, so the worker can pull per-bucket stats from the
# RGW admin API — michael is a plain S3 user without those caps.
radosEndpoint: http://rook-ceph-rgw-yucca.rook-ceph.svc:80
radosSecretName: rook-ceph-object-user-yucca-metrics
# Static dev secrets (overridden in dev via the yucca-dev-env extraEnvFrom
# layer, in prod via ExternalSecret)
secretData:
POLAR_ACCESS_TOKEN: "polar access token"
POLAR_WEBHOOK_SECRET: "polar webhook secret"
POLAR_PRODUCT_ID: "polar product ID"
POLAR_APPLY_DISCOUNT_ID: "polar apply discount ID"
extraEnvFrom: []
env:
- name: NODE_ENV
value: development
- name: YUCCA_METRICS_WORKER_PORT
value: "3040"
- name: LOG_LEVEL
value: debug
- name: OTEL_METRICS
value: http://victoria-metrics:8428/opentelemetry/v1/metrics
- name: OTEL_LOGGING
value: http://victoria-logs:9428/insert/opentelemetry/v1/logs
# Probes keep `tilt ci`/Flux honest: without them a crash-looping dev process
# still counts as Ready. The startupProbe budgets for the dev watcher's first
# boot; no livenessProbe so dev never restart-loops.
startupProbe:
tcpSocket: { port: http }
periodSeconds: 5
failureThreshold: 60
readinessProbe:
tcpSocket: { port: http }
periodSeconds: 10
+2
View File
@@ -4,10 +4,12 @@ resources:
- ./namespace.yaml
- ./database/ks.yaml
- ./object-user/ks.yaml
- ./metrics-object-user/ks.yaml
- ./mock-oidc/ks.yaml
- ./victoria-metrics/ks.yaml
- ./victoria-logs/ks.yaml
- ./michael/ks.yaml
- ./metrics-worker/ks.yaml
- ./yucca-api/ks.yaml
- ./yucca-admin-api/ks.yaml
- ./web/ks.yaml
@@ -0,0 +1,30 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-metrics-object-user
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/ceph-objectuser
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
remediation:
retries: 3
values:
# Dedicated RGW user for the metrics worker. michael is a plain S3 user with
# no admin caps, so it can't read per-bucket stats via the RGW admin API.
# Rook writes this user's keys to rook-ceph-object-user-yucca-metrics.
userName: metrics
capabilities:
bucket: read
usage: read
metadata: read
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,22 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-metrics-object-user
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-metrics-object-user
path: ./kubernetes/apps/yucca/metrics-object-user/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: rook-ceph-cluster
@@ -0,0 +1,26 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: yucca-metrics-worker
namespace: yucca
spec:
interval: 1h
chart:
spec:
chart: charts/yucca-metrics-worker
sourceRef:
kind: GitRepository
name: yucca
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
values:
image:
repository: ghcr.io/immich-app/yucca/yucca-metrics-worker # TODO: confirm prod registry
tag: 0.0.1
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
@@ -0,0 +1,23 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: yucca-metrics-worker
namespace: flux-system
spec:
targetNamespace: yucca
commonMetadata:
labels:
app.kubernetes.io/name: yucca-metrics-worker
path: ./kubernetes/apps/yucca/metrics-worker/app
prune: true
wait: true
interval: 1h
retryInterval: 2m
timeout: 5m
sourceRef:
kind: GitRepository
name: yucca
dependsOn:
- name: yucca-database
- name: yucca-object-user
@@ -551,6 +551,24 @@
"sizeBytes"
]
},
"RepositoryMeterDto": {
"type": "object",
"properties": {
"sizeBytes": {
"type": "number"
},
"objectCount": {
"type": "number"
},
"lastUpdated": {
"type": "string"
}
},
"required": [
"sizeBytes",
"objectCount"
]
},
"RepositoryWithMetricsDto": {
"type": "object",
"properties": {
@@ -565,6 +583,9 @@
},
"metrics": {
"$ref": "#/components/schemas/RepositoryMetricsDto"
},
"meter": {
"$ref": "#/components/schemas/RepositoryMeterDto"
}
},
"required": [
@@ -51,11 +51,17 @@ export type RepositoryMetricsDto = {
lastBackupDuration?: number;
sizeBytes: number;
};
export type RepositoryMeterDto = {
sizeBytes: number;
objectCount: number;
lastUpdated?: string;
};
export type RepositoryWithMetricsDto = {
id: string;
worm: boolean;
name: string;
metrics: RepositoryMetricsDto;
meter?: RepositoryMeterDto;
};
export type RepositoryCreateResponseDto = {
repository: RepositoryWithMetricsDto;
@@ -26,9 +26,23 @@ export class RepositoryMetricsDto {
sizeBytes!: number;
}
export class RepositoryMeterDto {
@ApiProperty()
sizeBytes!: number;
@ApiProperty()
objectCount!: number;
@ApiProperty({ type: 'string', required: false })
lastUpdated!: Date | null;
}
export class RepositoryWithMetricsDto extends RepositoryDto {
@ApiProperty()
metrics!: RepositoryMetricsDto;
@ApiProperty({ type: RepositoryMeterDto, required: false })
meter?: RepositoryMeterDto;
}
export class RepositoryCreateRequestDto {
@@ -13,6 +13,13 @@ const metricsJson = (eb: ExpressionBuilder<DB, 'repositories' | 'repositoryMetri
lastBackupDuration: eb.ref('repositoryMetrics.lastBackupDuration'),
}).as('metrics');
const meterJson = (eb: ExpressionBuilder<DB, 'repositories' | 'repositoryMeter'>) =>
jsonBuildObject({
sizeBytes: eb.fn.coalesce('repositoryMeter.sizeBytes', eb.val(0)),
objectCount: eb.fn.coalesce('repositoryMeter.objectCount', eb.val(0)),
lastUpdated: eb.ref('repositoryMeter.timestamp'),
}).as('meter');
@Injectable()
export class RepositoryRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@@ -26,9 +33,11 @@ export class RepositoryRepository {
return this.db
.selectFrom('repositories')
.leftJoin('repositoryMetrics', 'repositoryMetrics.id', 'repositories.id')
.leftJoin('repositoryMeter', 'repositoryMeter.repositoryId', 'repositories.id')
.where('repositories.id', '=', id)
.selectAll('repositories')
.select(metricsJson)
.select(meterJson)
.executeTakeFirstOrThrow();
}
@@ -36,9 +45,11 @@ export class RepositoryRepository {
return this.db
.selectFrom('repositories')
.leftJoin('repositoryMetrics', 'repositoryMetrics.id', 'repositories.id')
.leftJoin('repositoryMeter', 'repositoryMeter.repositoryId', 'repositories.id')
.where('userId', '=', userId)
.selectAll('repositories')
.select(metricsJson)
.select(meterJson)
.execute();
}
+13 -1
View File
@@ -1,5 +1,7 @@
import { Database } from '@immich/sql-tools';
import { RepositoryTable } from './tables/repository.table';
import { RepositoryMeterTable } from './tables/repositoryMeter.table';
import { RepositoryMeterHistoryTable } from './tables/repositoryMeterHistory.table';
import { RepositoryMetricsTable } from './tables/repositoryMetrics.table';
import { RepositoryMetricsHistoryTable } from './tables/repositoryMetricsHistory.table';
import { SessionTable } from './tables/session.table';
@@ -7,7 +9,15 @@ import { UserTable } from './tables/user.table';
@Database({ name: 'yucca' })
export class ImmichDatabase {
tables = [SessionTable, UserTable, RepositoryTable, RepositoryMetricsTable, RepositoryMetricsHistoryTable];
tables = [
SessionTable,
UserTable,
RepositoryTable,
RepositoryMetricsTable,
RepositoryMetricsHistoryTable,
RepositoryMeterTable,
RepositoryMeterHistoryTable,
];
functions = [];
@@ -20,4 +30,6 @@ export interface DB {
repositories: RepositoryTable;
repositoryMetrics: RepositoryMetricsTable;
repositoryMetricsHistory: RepositoryMetricsHistoryTable;
repositoryMeter: RepositoryMeterTable;
repositoryMeterHistory: RepositoryMeterHistoryTable;
}
@@ -0,0 +1,31 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`CREATE TABLE "repositoryMeter" (
"repositoryId" uuid NOT NULL,
"sizeBytes" bigint NOT NULL,
"objectCount" bigint NOT NULL,
"timestamp" timestamp with time zone NOT NULL DEFAULT now(),
CONSTRAINT "repositoryMeter_repositoryId_fkey" FOREIGN KEY ("repositoryId") REFERENCES "repositories" ("id") ON UPDATE CASCADE ON DELETE CASCADE,
CONSTRAINT "repositoryMeter_pkey" PRIMARY KEY ("repositoryId")
);`.execute(db);
await sql`CREATE TABLE "repositoryMeterHistory" (
"id" uuid NOT NULL DEFAULT gen_random_uuid(),
"repositoryId" uuid NOT NULL,
"sizeBytes" bigint NOT NULL,
"objectCount" bigint NOT NULL,
"timestamp" timestamp with time zone NOT NULL DEFAULT now(),
CONSTRAINT "repositoryMeterHistory_repositoryId_fkey" FOREIGN KEY ("repositoryId") REFERENCES "repositories" ("id") ON UPDATE CASCADE ON DELETE CASCADE,
CONSTRAINT "repositoryMeterHistory_pkey" PRIMARY KEY ("id")
);`.execute(db);
await sql`CREATE INDEX "repositoryMeterHistory_repositoryId_idx" ON "repositoryMeterHistory" ("repositoryId");`.execute(
db,
);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`DROP TABLE "repositoryMeterHistory";`.execute(db);
await sql`DROP TABLE "repositoryMeter";`.execute(db);
}
@@ -0,0 +1,17 @@
import { Column, ForeignKeyColumn, type Generated, Table } from '@immich/sql-tools';
import { RepositoryTable } from './repository.table';
@Table({ name: 'repositoryMeter' })
export class RepositoryMeterTable {
@ForeignKeyColumn(() => RepositoryTable, { primary: true, onUpdate: 'CASCADE', onDelete: 'CASCADE' })
repositoryId!: string;
@Column({ type: 'bigint' })
sizeBytes!: number;
@Column({ type: 'bigint' })
objectCount!: number;
@Column({ type: 'timestamp with time zone', default: () => 'now()' })
timestamp!: Generated<Date>;
}
@@ -0,0 +1,20 @@
import { Column, ForeignKeyColumn, type Generated, Table } from '@immich/sql-tools';
import { RepositoryTable } from './repository.table';
@Table({ name: 'repositoryMeterHistory' })
export class RepositoryMeterHistoryTable {
@Column({ primary: true, type: 'uuid', default: () => 'gen_random_uuid()' })
id!: Generated<string>;
@ForeignKeyColumn(() => RepositoryTable, { onUpdate: 'CASCADE', onDelete: 'CASCADE' })
repositoryId!: string;
@Column({ type: 'bigint' })
sizeBytes!: number;
@Column({ type: 'bigint' })
objectCount!: number;
@Column({ type: 'timestamp with time zone', default: () => 'now()' })
timestamp!: Generated<Date>;
}
@@ -141,6 +141,7 @@ describe('MetricsController (e2e)', () => {
lastSuccessfulBackup: expect.any(String),
lastBackupDuration: 1234,
}),
meter: expect.any(Object),
},
});
});
@@ -51,6 +51,7 @@ describe('RepositoryController (e2e)', () => {
worm: false,
name: 'My Repository',
metrics: expect.any(Object),
meter: expect.any(Object),
},
});
});
@@ -70,6 +71,7 @@ describe('RepositoryController (e2e)', () => {
worm: false,
name: expect.any(String),
metrics: expect.any(Object),
meter: expect.any(Object),
},
});
});
@@ -107,6 +109,7 @@ describe('RepositoryController (e2e)', () => {
worm: false,
name: 'Updated Name',
metrics: expect.any(Object),
meter: expect.any(Object),
},
});
});
+56
View File
@@ -0,0 +1,56 @@
# compiled output
/dist
/node_modules
/build
# Logs
logs
*.log
npm-debug.log*
pnpm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# OS
.DS_Store
# Tests
/coverage
/.nyc_output
# IDEs and editors
/.idea
.project
.classpath
.c9/
*.launch
.settings/
*.sublime-workspace
# IDE - VSCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# temp directory
.temp
.tmp
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+88
View File
@@ -0,0 +1,88 @@
# yucca-metrics-worker Docker image
# Built on/for Alpine Linux
#
# mise is used as a command runner only
# node.js & pnpm pinned in image =(
# (node.js 26 will remove yarn v1 fixing corepack install)
#
# References:
# ===========
# https://docs.nestjs.com/deployment
# https://mise.jdx.dev/mise-cookbook/docker.html
# https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md
# https://pnpm.io/cli/deploy
ARG ALPINE_VERSION=3.23
# Pinned alpine runtime base. NOTE: this digest is alpine:3.23's, so it must NOT
# be appended to the node/golang base tags (which reuse ${ALPINE_VERSION}).
ARG ALPINE_IMAGE=alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11
# Manifest stage: strips everything except package.jsons + workspace files so
# the pnpm-install layer below is cached on lockfile/manifest changes only.
FROM ${ALPINE_IMAGE} AS manifests
WORKDIR /src
COPY . ./
RUN find . -type d -name node_modules -prune -exec rm -rf {} + && \
find . -type f \
! -name 'package.json' \
! -name 'pnpm-lock.yaml' \
! -name 'pnpm-workspace.yaml' \
! -name '.npmrc' \
-delete && \
find . -type d -empty -delete
FROM node:25-alpine${ALPINE_VERSION} AS dev
WORKDIR /app
RUN apk add --no-cache bash && npm install -g pnpm@10.28.1
ENV NODE_ENV="development"
# 1. Install deps — cached unless lockfile or any package.json changes
COPY --from=manifests /src ./
RUN pnpm install --frozen-lockfile
# 2. Copy sources, build workspace libs yucca-metrics-worker imports at runtime
COPY . ./
RUN pnpm --filter @common/server build
EXPOSE 3040
CMD ["pnpm", "--filter", "yucca-metrics-worker", "start:dev"]
FROM node:25-alpine${ALPINE_VERSION} AS builder
WORKDIR /build-stage
COPY . ./
RUN apk add --no-cache curl bash
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV MISE_DATA_DIR="/mise"
ENV MISE_CONFIG_DIR="/mise"
ENV MISE_CACHE_DIR="/mise/cache"
ENV MISE_INSTALL_PATH="/usr/local/bin/mise"
ENV MISE_TASK_RUN_AUTO_INSTALL=false
ENV PATH="/mise/shims:$PATH"
ENV NODE_ENV="production"
RUN npm install -g pnpm@10.28.1
RUN curl https://mise.run | sh
RUN mise trust
RUN pnpm i --frozen-lockfile
RUN mise yucca-metrics-worker:build
RUN pnpm --filter yucca-metrics-worker deploy /deploy --prod --legacy
FROM ${ALPINE_IMAGE}
WORKDIR /usr/src/app
RUN apk add --no-cache libstdc++ dumb-init \
&& addgroup -g 1000 node && adduser -u 1000 -G node -s /bin/sh -D node \
&& chown node:node ./
COPY --from=builder /usr/local/bin/node /usr/local/bin/
COPY --from=builder /usr/local/bin/docker-entrypoint.sh /usr/local/bin/
ENTRYPOINT ["docker-entrypoint.sh"]
USER node
COPY --from=builder /deploy ./
ENV NODE_ENV="production"
EXPOSE 3040
CMD ["dumb-init", "node", "dist/main"]
@@ -0,0 +1,8 @@
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true
}
}
@@ -0,0 +1,74 @@
{
"name": "yucca-metrics-worker",
"version": "0.0.1",
"description": "",
"author": "",
"private": true,
"license": "UNLICENSED",
"scripts": {
"build": "nest build",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/main",
"test": "jest",
"test:cov": "jest --coverage",
"test:integration": "jest --config ./test/jest-integration.json --runInBand"
},
"files": [
"dist"
],
"dependencies": {
"@common/server": "workspace:^",
"@immich/sql-tools": "catalog:",
"@nestjs/common": "catalog:",
"@nestjs/core": "catalog:",
"@nestjs/platform-express": "catalog:",
"@nestjs/schedule": "catalog:",
"@opentelemetry/api": "catalog:",
"aws4fetch": "catalog:",
"kysely": "catalog:",
"kysely-postgres-js": "catalog:",
"nestjs-kysely": "catalog:",
"postgres": "catalog:",
"reflect-metadata": "catalog:",
"rxjs": "catalog:",
"zod": "catalog:"
},
"devDependencies": {
"@nestjs/cli": "catalog:",
"@nestjs/schematics": "catalog:",
"@nestjs/testing": "catalog:",
"@types/jest": "catalog:",
"@types/node": "catalog:",
"globals": "catalog:",
"jest": "catalog:",
"source-map-support": "catalog:",
"ts-jest": "catalog:",
"ts-loader": "catalog:",
"ts-node": "catalog:",
"tsconfig-paths": "catalog:",
"typescript": "catalog:",
"typescript-eslint": "catalog:"
},
"jest": {
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"moduleNameMapper": {
"^src/(.*)$": "<rootDir>/$1"
},
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
}
}
@@ -0,0 +1,28 @@
import { LoggerRepository, LoggingInterceptor, OtelModule, WideContextRepository } from '@common/server/otel';
import { Module } from '@nestjs/common';
import { APP_INTERCEPTOR } from '@nestjs/core';
import { ScheduleModule } from '@nestjs/schedule';
import { KyselyModule } from 'nestjs-kysely';
import { MeterRepository } from './repositories/meter.repository';
import { RepositoryRepository } from './repositories/repository.repository';
import { RgwRepository } from './repositories/rgw.repository';
import { MetricsService } from './services/metrics.service';
import { getKyselyConfig } from './utils/database';
export const imports = [KyselyModule.forRoot(getKyselyConfig()), ScheduleModule.forRoot()];
export const providers = [
WideContextRepository,
LoggerRepository,
RgwRepository,
MeterRepository,
RepositoryRepository,
MetricsService,
{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor },
];
@Module({
imports: [OtelModule, ...imports],
providers,
})
export class AppModule {}
+20
View File
@@ -0,0 +1,20 @@
import { z } from 'zod';
const schema = z.object({
NODE_ENV: z.enum(['development', 'production', 'test', 'provision']).default('development'),
YUCCA_METRICS_WORKER_PORT: z.coerce.number().min(1000),
POSTGRES_HOST: z.string(),
POSTGRES_PORT: z.coerce.number().default(5432),
POSTGRES_USERNAME: z.string(),
POSTGRES_PASSWORD: z.string(),
POSTGRES_DATABASE: z.string(),
POSTGRES_SSL: z.union([z.enum(['require', 'allow', 'prefer', 'verify-full']), z.boolean()]).default(false),
RADOS_ENDPOINT: z.url().transform((url) => new URL(url)),
RADOS_ACCESS_KEY_ID: z.string(),
RADOS_SECRET_ACCESS_KEY: z.string(),
});
export const env = schema.parse(process.env);
+12
View File
@@ -0,0 +1,12 @@
import '@common/server/otel';
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
import { env } from './env';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
await app.listen(env.YUCCA_METRICS_WORKER_PORT);
}
void bootstrap();
@@ -0,0 +1,25 @@
import { Injectable } from '@nestjs/common';
import { Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB } from 'src/schema';
type MeterReading = { sizeBytes: number; objectCount: number };
@Injectable()
export class MeterRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
record(repositoryId: string, { sizeBytes, objectCount }: MeterReading) {
const row = { repositoryId, sizeBytes, objectCount, timestamp: new Date() };
return this.db.transaction().execute(async (tx) => {
await tx
.insertInto('repositoryMeter')
.values(row)
.onConflict((oc) => oc.column('repositoryId').doUpdateSet({ sizeBytes, objectCount, timestamp: row.timestamp }))
.execute();
await tx.insertInto('repositoryMeterHistory').values(row).execute();
});
}
}
@@ -0,0 +1,13 @@
import { Injectable } from '@nestjs/common';
import { Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB } from 'src/schema';
@Injectable()
export class RepositoryRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
getAll() {
return this.db.selectFrom('repositories').select(['id', 'userId']).execute();
}
}
@@ -0,0 +1,69 @@
import { Injectable } from '@nestjs/common';
import { AwsClient } from 'aws4fetch';
import { env } from 'src/env';
type RgwUsageCategory = { num_objects?: number; size_actual?: number };
type RgwBucketEntry = { bucket: string; usage?: Record<string, RgwUsageCategory> };
export type BucketStats = { bucket: string; objects: number; bytes: number };
@Injectable()
export class RgwRepository {
private aws = new AwsClient({
accessKeyId: env.RADOS_ACCESS_KEY_ID,
secretAccessKey: env.RADOS_SECRET_ACCESS_KEY,
service: 's3',
region: 'rgw',
});
async *getBucketStats(): AsyncGenerator<BucketStats> {
const entries: RgwBucketEntry[] = await this.adminRequest('/admin/bucket', { format: 'json', stats: 'true' });
for (const entry of entries) {
yield {
bucket: entry.bucket,
objects: entry.usage?.['rgw.main']?.num_objects ?? 0,
bytes: entry.usage?.['rgw.main']?.size_actual ?? 0,
};
}
}
async *getBucketStatsStream(pageSize = 1000): AsyncGenerator<BucketStats> {
let marker = '';
for (;;) {
const query: Record<string, string> = { format: 'json', stats: 'true', 'max-entries': String(pageSize) };
if (marker) {
query.marker = marker;
}
const entries: RgwBucketEntry[] = await this.adminRequest('/admin/bucket', query);
for (const entry of entries) {
yield {
bucket: entry.bucket,
objects: entry.usage?.['rgw.main']?.num_objects ?? 0,
bytes: entry.usage?.['rgw.main']?.size_actual ?? 0,
};
}
const last = entries.at(-1);
if (!last || entries.length < pageSize) {
return;
}
marker = last.bucket;
}
}
private async adminRequest(path: string, query: Record<string, string>): Promise<any> {
const url = new URL(env.RADOS_ENDPOINT.href);
url.pathname = path;
url.search = new URLSearchParams(query).toString();
const response = await this.aws.fetch(url.toString());
if (!response.ok) {
throw new Error(`RGW admin ${path} failed: ${response.status} ${response.statusText} — ${await response.text()}`);
}
return response.json();
}
}
+1
View File
@@ -0,0 +1 @@
../../yucca-api/src/schema
@@ -0,0 +1,65 @@
import { LoggerRepository, MetricService } from '@common/server/otel';
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { Gauge } from '@opentelemetry/api';
import { env } from 'src/env';
import { MeterRepository } from 'src/repositories/meter.repository';
import { RepositoryRepository } from 'src/repositories/repository.repository';
import { RgwRepository } from 'src/repositories/rgw.repository';
@Injectable()
export class MetricsService implements OnApplicationBootstrap {
private readonly repositorySizeBytes: Gauge;
private readonly repositoryObjectCount: Gauge;
constructor(
private logger: LoggerRepository,
private rgw: RgwRepository,
private meter: MeterRepository,
private repositories: RepositoryRepository,
metricService: MetricService,
) {
this.repositorySizeBytes = metricService.getGauge('rgw_repository_size_bytes', {
description: 'Repository size in bytes',
});
this.repositoryObjectCount = metricService.getGauge('rgw_repository_object_count', {
description: 'Number of objects in the repository',
});
}
async onApplicationBootstrap() {
if (env.NODE_ENV === 'development') {
await this.sync();
}
}
@Cron(CronExpression.EVERY_5_MINUTES)
async sync() {
this.logger.info('Syncing from RadosGW...');
try {
const repositories = await this.repositories.getAll();
const customerByRepository = new Map(repositories.map((repository) => [repository.id, repository.userId]));
let count = 0;
for await (const { bucket: repositoryId, bytes, objects } of this.rgw.getBucketStats()) {
count++;
const customerId = customerByRepository.get(repositoryId);
if (!customerId) {
this.logger.warn(`RGW bucket "${repositoryId}" has no matching repository; skipping`);
continue;
}
await this.meter.record(repositoryId, { sizeBytes: bytes, objectCount: objects });
this.repositorySizeBytes.record(bytes, { repositoryId, customerId });
this.repositoryObjectCount.record(objects, { repositoryId, customerId });
}
this.logger.info(`Synced stats for ${count} buckets`);
} catch (error) {
this.logger.error(error, 'Failed to sync metrics from RadosGW');
}
}
}
@@ -0,0 +1,44 @@
import { KyselyConfig } from 'kysely';
import { PostgresJSDialect } from 'kysely-postgres-js';
import postgres, { Notice } from 'postgres';
import { env } from 'src/env';
export const getKyselyConnectionParameters = () => ({
connectionType: 'parts' as const,
host: env.POSTGRES_HOST,
port: env.POSTGRES_PORT,
username: env.POSTGRES_USERNAME,
password: env.POSTGRES_PASSWORD,
database: env.POSTGRES_DATABASE,
});
export const getKyselyConfig = (
options: Partial<postgres.Options<Record<string, postgres.PostgresType>>> = {},
): KyselyConfig => {
return {
dialect: new PostgresJSDialect({
postgres: postgres({
onnotice: (notice: Notice) => {
if (notice['severity'] !== 'NOTICE') {
console.warn('Postgres notice:', notice);
}
},
connection: {
TimeZone: 'UTC',
},
...getKyselyConnectionParameters(),
...options,
}),
}),
log(event) {
if (event.level === 'error') {
console.error('Query failed:', {
durationMs: event.queryDurationMillis,
error: event.error,
sql: event.query.sql,
params: event.query.parameters,
});
}
},
};
};
@@ -0,0 +1,13 @@
{
"moduleFileExtensions": ["js", "json", "ts"],
"rootDir": ".",
"testEnvironment": "node",
"testRegex": ".integration-spec.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"moduleNameMapper": {
"^src/(.*)$": "<rootDir>/../src/$1"
},
"forceExit": true
}
@@ -0,0 +1,66 @@
import { AwsClient } from 'aws4fetch';
import { env } from 'src/env';
import { type BucketStats, RgwRepository } from 'src/repositories/rgw.repository';
describe('RgwRepository (integration)', () => {
let repository: RgwRepository;
const s3 = new AwsClient({
accessKeyId: env.RADOS_ACCESS_KEY_ID,
secretAccessKey: env.RADOS_SECRET_ACCESS_KEY,
service: 's3',
region: 'rgw',
});
const suffix = `${Date.now()}`;
const buckets = [`yucca-it-${suffix}-a`, `yucca-it-${suffix}-b`];
const bucketUrl = (bucket: string) => new URL(`/${bucket}`, env.RADOS_ENDPOINT).toString();
beforeAll(async () => {
repository = new RgwRepository();
for (const bucket of buckets) {
const response = await s3.fetch(bucketUrl(bucket), { method: 'PUT' });
if (!response.ok) {
throw new Error(`Failed to create bucket ${bucket}: ${response.status} ${await response.text()}`);
}
}
});
afterAll(async () => {
for (const bucket of buckets) {
await s3.fetch(bucketUrl(bucket), { method: 'DELETE' });
}
});
it('gets stats for every bucket', async () => {
const stats: BucketStats[] = [];
for await (const stat of repository.getBucketStats()) {
stats.push(stat);
expect(typeof stat.bucket).toBe('string');
expect(stat.bucket.length).toBeGreaterThan(0);
expect(stat.objects).toBeGreaterThanOrEqual(0);
expect(stat.bytes).toBeGreaterThanOrEqual(0);
}
expect(stats.map((stat) => stat.bucket)).toEqual(expect.arrayContaining(buckets));
});
it('gets stats for every bucket with pagination', async () => {
const bulk: BucketStats[] = [];
for await (const stat of repository.getBucketStats()) {
bulk.push(stat);
}
const streamed: BucketStats[] = [];
for await (const stat of repository.getBucketStatsStream(1)) {
streamed.push(stat);
expect(stat.objects).toBeGreaterThanOrEqual(0);
expect(stat.bytes).toBeGreaterThanOrEqual(0);
}
expect(streamed.map((stat) => stat.bucket)).toEqual(expect.arrayContaining(buckets));
expect(new Set(streamed.map((stat) => stat.bucket))).toEqual(new Set(bulk.map((stat) => stat.bucket)));
});
});
@@ -0,0 +1,4 @@
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
}
@@ -0,0 +1,29 @@
{
"compilerOptions": {
"module": "nodenext",
"moduleResolution": "nodenext",
"resolvePackageJsonExports": true,
"esModuleInterop": true,
"isolatedModules": true,
"declaration": true,
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
"target": "ES2023",
"sourceMap": true,
"outDir": "./dist",
"incremental": true,
"skipLibCheck": true,
"strictNullChecks": true,
"forceConsistentCasingInFileNames": true,
"noImplicitAny": true,
"strictBindCallApply": false,
"noFallthroughCasesInSwitch": false,
"paths": {
"src/*": ["./src/*"]
},
"types": ["jest"]
},
"include": ["src", "test"]
}
@@ -1468,6 +1468,24 @@
"sizeBytes"
]
},
"RepositoryMeterDto": {
"type": "object",
"properties": {
"sizeBytes": {
"type": "number"
},
"objectCount": {
"type": "number"
},
"lastUpdated": {
"type": "string"
}
},
"required": [
"sizeBytes",
"objectCount"
]
},
"RepositoryBackendDto": {
"type": "object",
"properties": {
@@ -1546,6 +1564,9 @@
"metrics": {
"$ref": "#/components/schemas/RepositoryMetricsDto"
},
"meter": {
"$ref": "#/components/schemas/RepositoryMeterDto"
},
"backends": {
"$ref": "#/components/schemas/RepositoryBackendsDto"
},
@@ -1656,6 +1677,9 @@
"metrics": {
"$ref": "#/components/schemas/RepositoryMetricsDto"
},
"meter": {
"$ref": "#/components/schemas/RepositoryMeterDto"
},
"backends": {
"$ref": "#/components/schemas/RepositoryBackendsDto"
},
@@ -51,9 +51,23 @@ export class RepositoryMetricsDto {
sizeBytes!: number;
}
export class RepositoryMeterDto {
@ApiProperty({ type: Number })
sizeBytes!: number;
@ApiProperty({ type: Number })
objectCount!: number;
@ApiProperty({ type: String, required: false })
lastUpdated?: string;
}
export class RepositoryWithMetricsDto extends RepositoryDto {
@ApiProperty({ type: RepositoryMetricsDto })
metrics!: RepositoryMetricsDto;
@ApiProperty({ type: RepositoryMeterDto, required: false })
meter?: RepositoryMeterDto;
}
export class RepositoryBackendDto {
@@ -2,8 +2,8 @@
import type { LocalRepositoryDto } from "$lib/fetch-client";
import { getRepositoryActions } from "$lib/services/repository.service";
import { Badge, FormatBytes, Text } from "@immich/ui";
import RelativeTime from "../util/RelativeTime.svelte";
import StackListItem from "../ui/StackListItem.svelte";
import RelativeTime from "../util/RelativeTime.svelte";
type Props = {
repository: LocalRepositoryDto;
@@ -39,9 +39,15 @@
<Badge size="tiny" color="info">WORM</Badge>
{/if}
<Badge size="tiny" color="secondary">
<FormatBytes bytes={repository.metrics.sizeBytes} />
</Badge>
{#if repository.meter}
<Badge size="tiny" color="secondary">
<FormatBytes bytes={repository.meter?.sizeBytes} />
</Badge>
{:else}
<Badge size="tiny" color="secondary">
Estimated <FormatBytes bytes={repository.metrics.sizeBytes} />
</Badge>
{/if}
{#snippet trailing()}
{#if repository.metrics.lastBackup && (!repository.metrics.lastSuccessfulBackup || +new Date(repository.metrics.lastBackup) > +new Date(repository.metrics.lastSuccessfulBackup))}
@@ -50,7 +56,9 @@
</Badge>
{:else if repository.metrics.lastSuccessfulBackup}
<Badge size="tiny" color="success">
Successful <RelativeTime time={repository.metrics.lastSuccessfulBackup} />
Successful <RelativeTime
time={repository.metrics.lastSuccessfulBackup}
/>
</Badge>
{:else}
<Badge size="tiny" color="warning">Never backed up</Badge>
@@ -1,6 +1,6 @@
<script lang="ts">
import type { LocalRepositoryDto } from "$lib/fetch-client";
import { Card, CardBody, getByteUnitString } from "@immich/ui";
import { Badge, Card, CardBody, getByteUnitString } from "@immich/ui";
import VisualisationGauge from "../ui/VisualisationGauge.svelte";
type Props = {
@@ -10,10 +10,11 @@
const { repositories }: Props = $props();
const totalStored = $derived(
repositories.reduce(
(sum, repo) => sum + (repo.metrics?.sizeBytes ?? 0),
0,
),
repositories.reduce((sum, repo) => sum + (repo.meter?.sizeBytes ?? 0), 0),
);
const estimatedStored = $derived(
repositories.reduce((sum, repo) => sum + (repo.metrics?.sizeBytes ?? 0), 0),
);
</script>
@@ -22,6 +23,11 @@
<VisualisationGauge
title="Total Stored"
content={getByteUnitString(totalStored)}
/>
>
{#snippet subtitle()}
<Badge size="tiny">Estimated {getByteUnitString(estimatedStored)}</Badge
>
{/snippet}
</VisualisationGauge>
</CardBody>
</Card>
@@ -52,7 +52,11 @@
<Stack gap={0}>
<Heading size="tiny">Your library</Heading>
<Text>
<FormatBytes bytes={repository.metrics.sizeBytes} /> &middot;
{#if repository.meter}
Estimated <FormatBytes bytes={repository.meter.sizeBytes} />
{:else}
<FormatBytes bytes={repository.metrics.sizeBytes} />
{/if} &middot;
<span class="lowercase"
>{cronstrue.toString(schedule.cron, {
verbose: true,
@@ -111,6 +111,11 @@ export type RepositoryMetricsDto = {
lastBackupDuration?: number;
sizeBytes: number;
};
export type RepositoryMeterDto = {
sizeBytes: number;
objectCount: number;
lastUpdated?: string;
};
export type RepositoryBackendDto = {
id: string;
"type": BackendType;
@@ -129,6 +134,7 @@ export type LocalRepositoryDto = {
worm: boolean;
name: string;
metrics: RepositoryMetricsDto;
meter?: RepositoryMeterDto;
backends?: RepositoryBackendsDto;
configuration?: RepositoryConfigurationDto;
};
@@ -157,6 +163,7 @@ export type InspectedLocalRepositoryDto = {
worm: boolean;
name: string;
metrics: RepositoryMetricsDto;
meter?: RepositoryMeterDto;
backends?: RepositoryBackendsDto;
configuration?: RepositoryConfigurationDto;
snapshots: SnapshotDto[];
+99
View File
@@ -171,6 +171,9 @@ catalogs:
'@vitest/browser-playwright':
specifier: ^4.0.18
version: 4.0.18
aws4fetch:
specifier: ^1.0.20
version: 1.0.20
better-sqlite3:
specifier: ^12.6.2
version: 12.6.2
@@ -952,6 +955,97 @@ importers:
specifier: 'catalog:'
version: 5.9.3
packages/yucca-metrics-worker:
dependencies:
'@common/server':
specifier: workspace:^
version: link:../common
'@immich/sql-tools':
specifier: 'catalog:'
version: 0.3.2
'@nestjs/common':
specifier: 'catalog:'
version: 11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2)
'@nestjs/core':
specifier: 'catalog:'
version: 11.1.11(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@11.1.11)(@nestjs/websockets@11.1.14)(reflect-metadata@0.2.2)(rxjs@7.8.2)
'@nestjs/platform-express':
specifier: 'catalog:'
version: 11.1.11(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.11)
'@nestjs/schedule':
specifier: 'catalog:'
version: 6.1.1(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.11)
'@opentelemetry/api':
specifier: 'catalog:'
version: 1.9.0
aws4fetch:
specifier: 'catalog:'
version: 1.0.20
kysely:
specifier: 'catalog:'
version: 0.28.2
kysely-postgres-js:
specifier: 'catalog:'
version: 3.0.0(kysely@0.28.2)(postgres@3.4.8)
nestjs-kysely:
specifier: 'catalog:'
version: 3.1.2(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.11)(kysely@0.28.2)(reflect-metadata@0.2.2)
postgres:
specifier: 'catalog:'
version: 3.4.8
reflect-metadata:
specifier: 'catalog:'
version: 0.2.2
rxjs:
specifier: 'catalog:'
version: 7.8.2
zod:
specifier: 'catalog:'
version: 4.3.5
devDependencies:
'@nestjs/cli':
specifier: 'catalog:'
version: 11.0.14(@types/node@25.2.1)
'@nestjs/schematics':
specifier: 'catalog:'
version: 11.0.9(chokidar@4.0.3)(typescript@5.9.3)
'@nestjs/testing':
specifier: 'catalog:'
version: 11.1.11(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.11)(@nestjs/platform-express@11.1.11)
'@types/jest':
specifier: 'catalog:'
version: 30.0.0
'@types/node':
specifier: 'catalog:'
version: 25.2.1
globals:
specifier: 'catalog:'
version: 16.5.0
jest:
specifier: 'catalog:'
version: 30.2.0(@types/node@25.2.1)(ts-node@10.9.2(@types/node@25.2.1)(typescript@5.9.3))
source-map-support:
specifier: 'catalog:'
version: 0.5.21
ts-jest:
specifier: 'catalog:'
version: 29.4.6(@babel/core@7.28.5)(@jest/transform@30.2.0)(@jest/types@30.2.0)(babel-jest@30.2.0(@babel/core@7.28.5))(jest-util@30.2.0)(jest@30.2.0(@types/node@25.2.1)(ts-node@10.9.2(@types/node@25.2.1)(typescript@5.9.3)))(typescript@5.9.3)
ts-loader:
specifier: 'catalog:'
version: 9.5.4(typescript@5.9.3)(webpack@5.103.0)
ts-node:
specifier: 'catalog:'
version: 10.9.2(@types/node@25.2.1)(typescript@5.9.3)
tsconfig-paths:
specifier: 'catalog:'
version: 4.2.0
typescript:
specifier: 'catalog:'
version: 5.9.3
typescript-eslint:
specifier: 'catalog:'
version: 8.52.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3)
packages/yucca-sdk/orchestration-api:
dependencies:
'@futo-org/backups-api-client':
@@ -4001,6 +4095,9 @@ packages:
resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==}
engines: {node: '>=8.0.0'}
aws4fetch@1.0.20:
resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==}
axobject-query@4.1.0:
resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==}
engines: {node: '>= 0.4'}
@@ -10523,6 +10620,8 @@ snapshots:
atomic-sleep@1.0.0: {}
aws4fetch@1.0.20: {}
axobject-query@4.1.0: {}
babel-jest@30.2.0(@babel/core@7.28.5):
+1
View File
@@ -66,6 +66,7 @@ catalog:
'@types/tail': ^2.2.3
'@vitest/browser': ^4.0.18
'@vitest/browser-playwright': ^4.0.18
aws4fetch: ^1.0.20
better-sqlite3: ^12.6.2
class-transformer: ^0.5.1
class-validator: ^0.14.3