feat: data api (#57)

* feat: github data api

* wip: point at production for metrics queries (REVERT BEFORE MERGE)

* feat: use live data

* chore: backfill script wip

* chore: move metric prefixes to metrics repository

* fix: eslint config

* feat: backfill script

* chore: tweak ui

---------

Co-authored-by: Zack Pollard <zackpollard@ymail.com>

backend/.env.backfill

@@ -0,0 +1,6 @@
+export VMETRICS_DATA_WRITE_TOKEN="op://tf_$ENVIRONMENT/vmetrics_data_write_token/token"
+export VMETRICS_DATA_ADMIN_TOKEN="op://tf_$ENVIRONMENT/vmetrics_data_admin_token/token"
+export VMETRICS_DATA_API_URL="https://data.immich.cloud"
+export GITHUB_INSTALLATION_ID="op://tf/github_app_immich_read_only/installation_id"
+export GITHUB_APP_ID="op://tf/github_app_immich_read_only/app_id"
+export GITHUB_PEM_FILE="op://tf/github_app_immich_read_only/pkcs8"

backend/.eslintrc.cjs

@@ -1,19 +0,0 @@
-module.exports = {
-  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'],
-  parser: '@typescript-eslint/parser',
-  parserOptions: {
-    project: true,
-    sourceType: 'module',
-    tsconfigRootDir: __dirname,
-  },
-  plugins: ['@typescript-eslint', 'eslint-plugin-import'],
-  rules: {
-    curly: 2,
-    '@typescript-eslint/no-floating-promises': 'error',
-    '@typescript-eslint/no-explicit-any': 'off',
-    "no-restricted-imports": ["error", {
-      "patterns": [".*"]
-    }],
-  },
-  root: true,
-};

backend/eslint.config.mjs

@@ -0,0 +1,78 @@
+import { FlatCompat } from '@eslint/eslintrc';
+import js from '@eslint/js';
+import typescriptEslint from '@typescript-eslint/eslint-plugin';
+import tsParser from '@typescript-eslint/parser';
+import globals from 'globals';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all,
+});
+
+export default [
+  {
+    ignores: ['eslint.config.mjs'],
+  },
+  ...compat.extends(
+    'plugin:@typescript-eslint/recommended',
+  ),
+  {
+    plugins: {
+      '@typescript-eslint': typescriptEslint,
+    },
+
+    languageOptions: {
+      globals: {
+        ...globals.node,
+      },
+
+      parser: tsParser,
+      ecmaVersion: 5,
+      sourceType: 'module',
+
+      parserOptions: {
+        project: 'tsconfig.json',
+        tsconfigRootDir: __dirname,
+      },
+    },
+
+    rules: {
+      '@typescript-eslint/interface-name-prefix': 'off',
+      '@typescript-eslint/explicit-function-return-type': 'off',
+      '@typescript-eslint/explicit-module-boundary-types': 'off',
+      '@typescript-eslint/no-explicit-any': 'off',
+      '@typescript-eslint/no-floating-promises': 'error',
+      'unicorn/prevent-abbreviations': 'off',
+      'unicorn/filename-case': 'off',
+      'unicorn/no-null': 'off',
+      'unicorn/prefer-top-level-await': 'off',
+      'unicorn/prefer-event-target': 'off',
+      'unicorn/no-thenable': 'off',
+      'unicorn/import-style': 'off',
+      'unicorn/prefer-structured-clone': 'off',
+      '@typescript-eslint/await-thenable': 'error',
+      '@typescript-eslint/no-misused-promises': 'error',
+      'require-await': 'off',
+      '@typescript-eslint/require-await': 'error',
+      curly: 2,
+      'object-shorthand': ['error', 'always'],
+
+      'no-restricted-imports': [
+        'error',
+        {
+          patterns: [
+            {
+              group: ['.*'],
+              message: 'Relative imports are not allowed.',
+            },
+          ],
+        },
+      ],
+    },
+  },
+];

backend/package-lock.json

@@ -15,7 +15,8 @@
         "fetch-retry": "^6.0.0",
         "fflate": "^0.8.2",
         "itty-router": "^5.0.18",
-        "p-limit": "^6.1.0"
+        "luxon": "^3.5.0",
+        "p-limit": "^6.2.0"
       },
       "devDependencies": {
         "@cloudflare/vitest-pool-workers": "^0.5.8",
@@ -23,6 +24,7 @@
         "@octokit/types": "^13.6.2",
         "@octokit/webhooks-types": "^7.6.1",
         "@types/express": "^4.17.21",
+        "@types/luxon": "^3.4.2",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
         "@typescript-eslint/parser": "^8.0.0",
         "@vitest/coverage-istanbul": "^2.1.1",
@@ -3456,6 +3458,13 @@
       "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
       "dev": true
     },
+    "node_modules/@types/luxon": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz",
+      "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/mime": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -6933,6 +6942,15 @@
         "yallist": "^3.0.2"
       }
     },
+    "node_modules/luxon": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/magic-string": {
       "version": "0.30.12",
       "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz",
@@ -7412,9 +7430,9 @@
       "dev": true
     },
     "node_modules/p-limit": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz",
-      "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==",
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz",
+      "integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==",
       "license": "MIT",
       "dependencies": {
         "yocto-queue": "^1.1.1"

backend/package.json

@@ -2,6 +2,7 @@
   "name": "worker",
   "version": "0.0.0",
   "private": true,
+  "type": "module",
   "scripts": {
     "deploy": "wrangler deploy",
     "dev": "wrangler dev",
@@ -10,7 +11,7 @@
     "test": "vitest",
     "test:cov": "vitest --coverage",
     "cf-typegen": "wrangler types",
-    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\" --max-warnings 0",
+    "lint": "eslint \"src/**/*.ts\" --max-warnings 0",
     "lint:fix": "npm run lint -- --fix",
     "format": "prettier --check .",
     "format:fix": "prettier --write .",
@@ -22,6 +23,7 @@
     "@octokit/types": "^13.6.2",
     "@octokit/webhooks-types": "^7.6.1",
     "@types/express": "^4.17.21",
+    "@types/luxon": "^3.4.2",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "@vitest/coverage-istanbul": "^2.1.1",
@@ -43,7 +45,8 @@
     "fetch-retry": "^6.0.0",
     "fflate": "^0.8.2",
     "itty-router": "^5.0.18",
-    "p-limit": "^6.1.0"
+    "luxon": "^3.5.0",
+    "p-limit": "^6.2.0"
   },
   "volta": {
     "node": "22.12.0"

backend/src/bin/backfill.ts

@@ -0,0 +1,695 @@
+import { createAppAuth } from '@octokit/auth-app';
+import { graphql } from '@octokit/graphql';
+import { DateTime } from 'luxon';
+import { GithubRepo, GithubUser } from 'src/constants';
+import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provider.repository';
+import { MetricsPushRepository } from 'src/repositories/metrics-push.repository';
+import { GithubMetric } from 'src/workers/ingest-processor.worker';
+
+type PageInfo = { endCursor: string; hasNextPage: boolean };
+type Paginated<T> = T & { pageInfo: PageInfo };
+type GithubStarGazer = { starredAt: string; user: { login: string; id: number } };
+type TimelineEvent = { actor: GithubUser };
+type GithubIssue = {
+  createdAt: string;
+  closedAt?: string;
+  number: number;
+  stateReason: 'COMPLETED' | 'NOT_PLANNED' | 'REOPENED';
+  author: GithubUser;
+  timelineItems: {
+    nodes: TimelineEvent[];
+  };
+};
+type GithubPullRequest = {
+  createdAt: string;
+  closedAt?: string;
+  mergedAt?: string;
+  number: number;
+  state: 'CLOSED' | 'MERGED' | 'OPEN';
+  author: GithubUser;
+  timelineItems: {
+    nodes: TimelineEvent[];
+  };
+};
+type GithubDiscussion = {
+  createdAt: string;
+  closedAt?: string;
+  answerChosenAt?: string;
+  answerChosenBy?: GithubUser;
+  number: number;
+  category: { name: string };
+  stateReason: 'RESOLVED' | 'OUTDATED' | 'DUPLICATE' | 'REOPENED';
+  author: GithubUser;
+};
+
+const immichOrg = 'immich-app';
+
+const compare = (a: DateTime, b: DateTime) => {
+  if (a.toMillis() === b.toMillis()) {
+    return 0;
+  }
+
+  return a < b ? -1 : 1;
+};
+
+if (!process.env.GITHUB_APP_ID || !process.env.GITHUB_PEM_FILE || !process.env.GITHUB_INSTALLATION_ID) {
+  throw new Error('Missing GitHub App credentials');
+}
+
+if (
+  !process.env.VMETRICS_DATA_WRITE_TOKEN ||
+  !process.env.VMETRICS_DATA_API_URL ||
+  !process.env.VMETRICS_DATA_ADMIN_TOKEN
+) {
+  throw new Error('Missing VMetrics API credentials');
+}
+
+if (!process.env.ENVIRONMENT) {
+  throw new Error('Missing environment');
+}
+
+const env = process.env.ENVIRONMENT;
+
+const influxProvider = new InfluxMetricsPushProvider(
+  process.env.VMETRICS_DATA_API_URL,
+  process.env.VMETRICS_DATA_WRITE_TOKEN,
+);
+const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]);
+
+const auth = createAppAuth({
+  appId: process.env.GITHUB_APP_ID,
+  privateKey: process.env.GITHUB_PEM_FILE,
+  installationId: process.env.GITHUB_INSTALLATION_ID,
+});
+
+const graphqlWithAuth = graphql.defaults({
+  request: {
+    hook: auth.hook,
+  },
+});
+
+const deleteTimeSeries = async (name_pattern: string, environment: string, repo: GithubRepo) => {
+  const url = new URL(`${process.env.VMETRICS_DATA_API_URL}/api/v1/admin/tsdb/delete_series`);
+  const match = `{__name__=~'${name_pattern}',environment='${environment}',org_name='${repo.organization}',repository_name='${repo.name}'}`;
+  url.search = new URLSearchParams({ ['match[]']: match }).toString();
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: {
+      Authorization: `Bearer ${process.env.VMETRICS_DATA_ADMIN_TOKEN}`,
+      'Content-Type': 'application/json',
+    },
+  });
+  console.log(`Deleting time series ${match}`);
+  if (response.status != 204) {
+    console.log(response.status);
+    throw new Error(`Failed to delete time series ${match}`);
+  }
+};
+
+export class BackfillService {
+  async backfillStars(repo: GithubRepo) {
+    const stargazers = await this.getStargazers(repo);
+    console.log(`[${repo.name}] - Loaded ${stargazers.length} stargazers`);
+    const deleteTime = DateTime.now();
+
+    const items = stargazers
+      .map(({ starredAt, user }) => ({ starredAt: DateTime.fromISO(starredAt), user }))
+      .sort((a, b) => compare(a.starredAt, b.starredAt))
+      .filter(({ starredAt }) => starredAt < deleteTime);
+
+    let total = 0;
+
+    for (const { starredAt, user } of items) {
+      total++;
+      const metric = new GithubMetric('star');
+
+      metric
+        .withRepository(repo)
+        .withUser(user)
+        .setTimestamp(starredAt.toJSDate())
+        .intField('total', total)
+        .intField('count', 1)
+        .addTag('environment', env);
+
+      metricsRepository.push(metric);
+    }
+
+    await deleteTimeSeries('immich_data_repository_star_.*', env, repo);
+    await influxProvider.flush();
+
+    console.log(`[${repo.name}] - Flushed ${total} stars`);
+  }
+
+  async backfillIssues(repo: GithubRepo) {
+    const issues = await this.getIssues(repo);
+    console.log(`[${repo.name}] - Loaded ${issues.length} issues`);
+
+    await deleteTimeSeries('immich_data_repository_issue_.*', env, repo);
+    const deleteTime = DateTime.now();
+
+    const events: { timestamp: Date; type: string; data: GithubIssue }[] = [];
+
+    for (const issue of issues) {
+      const { createdAt, closedAt } = issue;
+      const created = DateTime.fromISO(createdAt);
+      const closed = closedAt ? DateTime.fromISO(closedAt) : undefined;
+
+      if (created > deleteTime) {
+        continue;
+      }
+      events.push({ timestamp: created.toJSDate(), type: 'open', data: issue });
+      if (closed && closed < deleteTime) {
+        events.push({ timestamp: closed.toJSDate(), type: 'closed', data: issue });
+      }
+    }
+
+    events.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+
+    let opened = 0;
+    let closed = 0;
+
+    const metricPromises = [];
+    let count = 0;
+
+    for (const {
+      timestamp,
+      type,
+      data: { author, timelineItems },
+    } of events) {
+      count++;
+      const metric = new GithubMetric('issue');
+      if (type === 'open') {
+        opened++;
+      } else {
+        closed++;
+      }
+
+      metric
+        .withRepository(repo)
+        .withUser(type === 'open' ? author : timelineItems.nodes[0].actor)
+        .setTimestamp(timestamp)
+        .intField('total', opened)
+        .intField('open_total', opened - closed)
+        .intField('closed_total', closed)
+        .intField('count', type === 'open' ? 1 : -1)
+        .addTag('environment', env);
+
+      metricsRepository.push(metric);
+    }
+
+    await influxProvider.flush();
+    console.log(`[${repo.name}] - Flushed ${count} issues`);
+  }
+
+  async backfillDiscussions(repo: GithubRepo) {
+    const discussions = await this.getDiscussions(repo);
+    console.log(`[${repo.name}] - Loaded ${discussions.length} discussions`);
+
+    await deleteTimeSeries('immich_data_repository_discussion_.*', env, repo);
+    const deleteTime = DateTime.now();
+
+    const events: { timestamp: Date; type: string; data: GithubDiscussion }[] = [];
+
+    for (const discussion of discussions) {
+      const { createdAt, closedAt, answerChosenAt } = discussion;
+      const created = DateTime.fromISO(createdAt);
+      const closed = closedAt ? DateTime.fromISO(closedAt) : undefined;
+      const answered = answerChosenAt ? DateTime.fromISO(answerChosenAt) : undefined;
+
+      if (created > deleteTime) {
+        continue;
+      }
+      events.push({ timestamp: created.toJSDate(), type: 'open', data: discussion });
+      if (closed && closed < deleteTime) {
+        events.push({ timestamp: closed.toJSDate(), type: 'closed', data: discussion });
+      }
+      if (answered && answered < deleteTime) {
+        events.push({ timestamp: answered.toJSDate(), type: 'answered', data: discussion });
+      }
+    }
+
+    events.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+
+    let opened = 0;
+    let closed = 0;
+    let answered = 0;
+    let count = 0;
+
+    for (const {
+      timestamp,
+      type,
+      data: { author, answerChosenBy },
+    } of events) {
+      count++;
+      const metric = new GithubMetric('discussion');
+      if (type === 'open') {
+        opened++;
+      } else if (type === 'closed') {
+        closed++;
+      } else if (type === 'answered') {
+        answered++;
+      }
+
+      metric
+        .withRepository(repo)
+        .withUser(type === 'answered' ? answerChosenBy : author)
+        .setTimestamp(timestamp)
+        .intField('total', opened)
+        .intField('open_total', opened - closed)
+        .intField('closed_total', closed)
+        .intField('answered_total', answered)
+        .intField('count', type === 'open' ? 1 : -1)
+        .addTag('environment', env);
+
+      metricsRepository.push(metric);
+    }
+
+    await influxProvider.flush();
+    console.log(`[${repo.name}] - Flushed ${count} discussions`);
+  }
+
+  async backfillPullRequests(repo: GithubRepo) {
+    const pullRequests = await this.getPullRequests(repo);
+    console.log(`[${repo.name}] - Loaded ${pullRequests.length} pull requests`);
+
+    await deleteTimeSeries('immich_data_repository_pull_request_.*', env, repo);
+    const deleteTime = DateTime.now();
+
+    const events: { timestamp: Date; type: string; data: GithubPullRequest }[] = [];
+
+    for (const pullRequest of pullRequests) {
+      const { createdAt, closedAt, mergedAt } = pullRequest;
+      const created = DateTime.fromISO(createdAt);
+      const closed = closedAt ? DateTime.fromISO(closedAt) : undefined;
+      const merged = mergedAt ? DateTime.fromISO(mergedAt) : undefined;
+
+      if (created > deleteTime) {
+        continue;
+      }
+      events.push({ timestamp: created.toJSDate(), type: 'open', data: pullRequest });
+      if (closed && !merged && closed < deleteTime) {
+        events.push({ timestamp: closed.toJSDate(), type: 'closed', data: pullRequest });
+      }
+      if (merged && merged < deleteTime) {
+        events.push({ timestamp: merged.toJSDate(), type: 'merged', data: pullRequest });
+      }
+    }
+
+    events.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
+
+    let opened = 0;
+    let closed = 0;
+    let merged = 0;
+    let count = 0;
+
+    for (const {
+      timestamp,
+      type,
+      data: { author, timelineItems },
+    } of events) {
+      count++;
+      const metric = new GithubMetric('pull_request');
+      if (type === 'open') {
+        opened++;
+      } else if (type === 'closed') {
+        closed++;
+      } else if (type === 'merged') {
+        merged++;
+      }
+
+      metric
+        .withRepository(repo)
+        .withUser(type === 'open' ? author : timelineItems.nodes[0].actor)
+        .setTimestamp(timestamp)
+        .intField('total', opened)
+        .intField('open_total', opened - closed - merged)
+        .intField('closed_total', closed)
+        .intField('merged_total', merged)
+        .intField('count', type === 'open' ? 1 : -1)
+        .addTag('environment', env);
+
+      metricsRepository.push(metric);
+    }
+
+    await influxProvider.flush();
+    console.log(`[${repo.name}] - Flushed ${count} pull requests`);
+  }
+
+  async getStargazers(repo: GithubRepo) {
+    const results: GithubStarGazer[] = [];
+    let cursor: string | undefined;
+    const org = repo.organization;
+
+    console.log(`[${repo.name}] - Fetching stargazers`);
+
+    do {
+      const { repository } = await graphqlWithAuth<{
+        repository: {
+          stargazers: Paginated<{ edges: GithubStarGazer[] }>;
+        };
+      }>(
+        `
+          query ($org: String!, $repo: String!, $take: Int!, $cursor: String) {
+            repository(owner: $org, name: $repo) {
+              stargazers(first: $take, after: $cursor, orderBy: { field: STARRED_AT, direction: ASC }) {
+                edges {
+                  starredAt
+                  user: node {
+                    login
+                    id: databaseId
+                  }
+                }
+                pageInfo {
+                  endCursor
+                  hasNextPage
+                }
+              }
+            }
+          }
+        `,
+        {
+          org,
+          repo: repo.name,
+          take: 100,
+          cursor,
+        },
+      );
+
+      results.push(...repository.stargazers.edges);
+      if (results.length % 1000 === 0) {
+        console.log(`[${repo.name}] - Progress: ${results.length.toLocaleString()} stargazers`);
+      }
+
+      const { hasNextPage, endCursor } = repository.stargazers.pageInfo;
+      cursor = hasNextPage ? endCursor : undefined;
+    } while (cursor);
+
+    return results;
+  }
+
+  async getIssues(repo: GithubRepo) {
+    const results: GithubIssue[] = [];
+    let cursor: string | undefined;
+    const org = repo.organization;
+
+    console.log(`[${repo.name}] - Fetching issues`);
+
+    do {
+      const { repository } = await graphqlWithAuth<{ repository: { issues: Paginated<{ nodes: GithubIssue[] }> } }>(
+        `
+          query ($org: String!, $repo: String!, $take: Int!, $cursor: String) {
+            repository(owner: $org, name: $repo) {
+              issues(first: $take, after: $cursor) {
+                pageInfo {
+                  endCursor
+                  hasNextPage
+                }
+                nodes {
+                  createdAt
+                  closedAt
+                  number
+                  stateReason
+                  author {
+                    login
+                    ... on User {
+                      id: databaseId
+                    }
+                    ... on Bot {
+                      id: databaseId
+                    }
+                  }
+                  timelineItems(itemTypes:[CLOSED_EVENT, CONVERTED_TO_DISCUSSION_EVENT], last: 1){
+                    nodes {
+                      ... on ClosedEvent {
+                        actor {
+                          login
+                          ... on User {
+                            id: databaseId
+                          }
+                          ... on Bot {
+                            id: databaseId
+                          }
+                        }
+                      }
+                      ... on ConvertedToDiscussionEvent {
+                        actor {
+                          login
+                          ... on User {
+                            id: databaseId
+                          }
+                          ... on Bot {
+                            id: databaseId
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        `,
+        {
+          org,
+          repo: repo.name,
+          take: 100,
+          cursor,
+        },
+      );
+
+      results.push(...repository.issues.nodes);
+      if (results.length % 1000 === 0) {
+        console.log(`[${repo.name}] - Progress: ${results.length.toLocaleString()} issues`);
+      }
+
+      const { hasNextPage, endCursor } = repository.issues.pageInfo;
+      cursor = hasNextPage ? endCursor : undefined;
+    } while (cursor);
+
+    return results;
+  }
+
+  async getPullRequests(repo: GithubRepo) {
+    const results: GithubPullRequest[] = [];
+    let cursor: string | undefined;
+    const org = repo.organization;
+
+    console.log(`[${repo.name}] - Fetching pull requests`);
+
+    do {
+      const { repository } = await graphqlWithAuth<{
+        repository: { pullRequests: Paginated<{ nodes: GithubPullRequest[] }> };
+      }>(
+        `
+          query ($org: String!, $repo: String!, $take: Int!, $cursor: String) {
+            repository(owner: $org, name: $repo) {
+              pullRequests(first: $take, after: $cursor) {
+                pageInfo {
+                  endCursor
+                  hasNextPage
+                }
+                nodes {
+                  createdAt
+                  closedAt
+                  mergedAt
+                  number
+                  state
+                  author {
+                    login
+                    ... on User {
+                      id: databaseId
+                    }
+                    ... on Bot {
+                      id: databaseId
+                    }
+                  }
+                  timelineItems(itemTypes: [CLOSED_EVENT, MERGED_EVENT], last: 1) {
+                    nodes {
+                      ... on ClosedEvent {
+                        actor {
+                          login
+                          ... on User {
+                            id: databaseId
+                          }
+                          ... on Bot {
+                            id: databaseId
+                          }
+                        }
+                      }
+                      ... on MergedEvent {
+                        actor {
+                          login
+                          ... on User {
+                            id: databaseId
+                          }
+                          ... on Bot {
+                            id: databaseId
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        `,
+        {
+          org,
+          repo: repo.name,
+          take: 100,
+          cursor,
+        },
+      );
+
+      results.push(...repository.pullRequests.nodes);
+      if (results.length % 1000 === 0) {
+        console.log(`[${repo.name}] - Progress: ${results.length.toLocaleString()} pull requests`);
+      }
+
+      const { hasNextPage, endCursor } = repository.pullRequests.pageInfo;
+      cursor = hasNextPage ? endCursor : undefined;
+    } while (cursor);
+
+    return results;
+  }
+
+  async getDiscussions(repo: GithubRepo) {
+    const results: GithubDiscussion[] = [];
+    let cursor: string | undefined;
+    const org = repo.organization;
+
+    console.log(`[${repo.name}] - Fetching discussions`);
+
+    do {
+      const { repository } = await graphqlWithAuth<{
+        repository: { discussions: Paginated<{ nodes: GithubDiscussion[] }> };
+      }>(
+        `
+          query ($org: String!, $repo: String!, $take: Int!, $cursor: String) {
+            repository(owner: $org, name: $repo) {
+              discussions(first: $take, after: $cursor) {
+                pageInfo {
+                  endCursor
+                  hasNextPage
+                }
+                nodes {
+                  createdAt
+                  closedAt
+                  answerChosenAt
+                  answerChosenBy {
+                    login
+                    ... on User {
+                      id: databaseId
+                    }
+                    ... on Bot {
+                      id: databaseId
+                    }
+                  }
+                  number
+                  category {
+                    name
+                  }
+                  stateReason
+                  author {
+                    login
+                    ... on User {
+                      id: databaseId
+                    }
+                    ... on Bot {
+                      id: databaseId
+                    }
+                  }
+                }
+              }
+            }
+          }
+        `,
+        {
+          org,
+          repo: repo.name,
+          take: 100,
+          cursor,
+        },
+      );
+
+      results.push(...repository.discussions.nodes);
+      if (results.length % 1000 === 0) {
+        console.log(`[${repo.name}] - Progress: ${results.length.toLocaleString()} discussions`);
+      }
+
+      const { hasNextPage, endCursor } = repository.discussions.pageInfo;
+      cursor = hasNextPage ? endCursor : undefined;
+    } while (cursor);
+
+    return results;
+  }
+
+  async getGithubReposForOrg(org: string): Promise<GithubRepo[]> {
+    const results: { name: string }[] = [];
+    let cursor: string | undefined;
+
+    console.log(`Fetching repositories for ${org}`);
+
+    do {
+      const { organization } = await graphqlWithAuth<{
+        organization: { repositories: Paginated<{ nodes: { name: string }[] }> };
+      }>(
+        `
+          query ($org: String!, $take: Int!, $cursor: String) {
+            organization(login: $org) {
+              repositories(first: $take, after: $cursor) {
+                pageInfo {
+                  endCursor
+                  hasNextPage
+                }
+                nodes {
+                  name
+                }
+              }
+            }
+          }
+        `,
+        {
+          org,
+          take: 100,
+          cursor,
+        },
+      );
+
+      results.push(...organization.repositories.nodes);
+      if (results.length % 1000 === 0) {
+        console.log(`Progress: ${results.length.toLocaleString()} discussions`);
+      }
+
+      const { hasNextPage, endCursor } = organization.repositories.pageInfo;
+      cursor = hasNextPage ? endCursor : undefined;
+    } while (cursor);
+
+    return results.map(({ name }) => ({
+      organization: org,
+      name,
+    }));
+  }
+}
+
+const main = async () => {
+  const service = new BackfillService();
+  const repos = await service.getGithubReposForOrg(immichOrg);
+  const promises: Promise<unknown>[] = [];
+  for (const repo of repos) {
+    promises.push(
+      ...[
+        service.backfillStars(repo),
+        service.backfillPullRequests(repo),
+        service.backfillIssues(repo),
+        service.backfillDiscussions(repo),
+      ],
+    );
+  }
+  await Promise.all(promises);
+};
+
+main()
+  .then(() => console.log('Done'))
+  .catch(console.error);

backend/src/constants.ts

@@ -1,4 +1,5 @@
 export type GithubRepo = { name: string; organization: string };
+export type GithubUser = { login: string; id: number };
 
 export const GithubRepos: { [key: string]: GithubRepo } = {
   IMMICH: { name: 'immich', organization: 'immich-app' },

backend/src/env.d.ts

@@ -3,8 +3,9 @@ interface WorkerEnv extends Omit<Env, 'ENVIRONMENT' | 'STAGE' | 'DEPLOYMENT_KEY'
   ENVIRONMENT: string;
   STAGE: string;
   DEPLOYMENT_KEY: string;
-  VMETRICS_API_TOKEN: string;
-  VMETRICS_API_URL: string;
+  VMETRICS_DATA_WRITE_TOKEN: string;
+  VMETRICS_DATA_READ_TOKEN: string;
+  VMETRICS_DATA_API_URL: string;
   GITHUB_APP_ID: string;
   GITHUB_APP_PEM: string;
   GITHUB_APP_INSTALLATION_ID: string;

backend/src/index.ts

@@ -1,27 +1,24 @@
 import { AutoRouter, error, IRequest } from 'itty-router';
 import { QueueItem } from 'src/interfaces/queue.interface';
 import { CloudflareDeferredRepository } from 'src/repositories/cloudflare-deterred.repository';
-import { CloudflareMetricsRepository } from 'src/repositories/cloudflare-metrics.repository';
 import { CloudflareQueueRepository } from 'src/repositories/cloudflare-queue.repository';
 import { GithubRepository } from 'src/repositories/github.repository';
-import { InfluxMetricsProvider } from 'src/repositories/influx-metrics-provider.repository';
+import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provider.repository';
+import { MetricsPushRepository } from 'src/repositories/metrics-push.repository';
+import { MetricsQueryRepository } from 'src/repositories/metrics-query.repository';
 import { ApiWorker } from 'src/workers/api.worker';
 import { IngestApiWorker } from 'src/workers/ingest-api.worker';
 import { IngestProcessorWorker } from 'src/workers/ingest-processor.worker';
 
 type FetchRequest = IRequest & Parameters<ExportedHandlerFetchHandler>[0];
 
-const asTags = (request: FetchRequest) => ({
-  continent: request.cf?.continent ?? '',
-  colo: request.cf?.colo ?? '',
-  asOrg: request.cf?.asOrganization ?? '',
-});
+const asEnvTag = (env: WorkerEnv) => [env.ENVIRONMENT, env.STAGE].filter(Boolean).join('_');
 
 const newApiWorker = (request: FetchRequest, env: WorkerEnv, ctx: ExecutionContext) => {
   const deferredRepository = new CloudflareDeferredRepository(ctx);
-  const influxProvider = new InfluxMetricsProvider(env.VMETRICS_API_URL, env.VMETRICS_API_TOKEN);
+  const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN);
   deferredRepository.defer(() => influxProvider.flush());
-  const metrics = new CloudflareMetricsRepository('data', asTags(request), [influxProvider]);
+  const metrics = new MetricsQueryRepository(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_READ_TOKEN, 'prod');
 
   return new ApiWorker(metrics);
 };
@@ -43,7 +40,7 @@ const handleError = (err: Error) => {
 };
 
 const router = AutoRouter<FetchRequest, [WorkerEnv, ExecutionContext]>()
-  .get('/api', (...args) => newApiWorker(...args).getGithubReports())
+  .get('/api/github', (...args) => newApiWorker(...args).getGithubReports())
   .post('/ingest/github/:slug', withSlug, async (req, env) =>
     newIngestApiWorker(req, env)
       .onGithubEvent(await req.json())
@@ -54,15 +51,14 @@ const router = AutoRouter<FetchRequest, [WorkerEnv, ExecutionContext]>()
 export default {
   fetch: router.fetch,
   queue: async (batch, env) => {
-    const influxProvider = new InfluxMetricsProvider(env.VMETRICS_API_URL, env.VMETRICS_API_TOKEN);
-    const metricsRepository = new CloudflareMetricsRepository('data', {}, [influxProvider]);
+    const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN);
+    const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]);
     const githubRepository = new GithubRepository(
       env.GITHUB_APP_ID,
       env.GITHUB_APP_PEM,
       env.GITHUB_APP_INSTALLATION_ID,
     );
-    const envTag = [env.ENVIRONMENT, env.STAGE].filter(Boolean).join('_');
-    const ingestProcessWorker = new IngestProcessorWorker(metricsRepository, githubRepository, envTag);
+    const ingestProcessWorker = new IngestProcessorWorker(metricsRepository, githubRepository, asEnvTag(env));
 
     if (batch.queue.startsWith('data-ingest')) {
       await ingestProcessWorker.handleMessages(batch);

backend/src/interfaces/github.interface.ts

@@ -29,6 +29,7 @@ export type DiscussionsCountsResponse = {
   open: number;
   closed: number;
   total: number;
+  answered: number;
 };
 
 export interface IGithubRepository {

backend/src/interfaces/metrics-provider.interface.ts

@@ -1,6 +1,6 @@
 import { Metric } from 'src/interfaces/metrics.interface';
 
-export interface IMetricsProviderRepository {
+export interface IMetricsPushProviderRepository {
   pushMetric(metric: Metric): void;
-  flush(): void;
+  flush(): Promise<void>;
 }

backend/src/interfaces/metrics.interface.ts

@@ -1,6 +1,6 @@
 import { AsyncFn, Operation, Options } from 'src/types';
 
-export interface IMetricsRepository {
+export interface IMetricsPushRepository {
   monitorAsyncFunction<T extends AsyncFn>(
     operation: Operation,
     call: T,
@@ -11,8 +11,9 @@ export interface IMetricsRepository {
 
 export class Metric {
   private _tags: Map<string, string> = new Map();
-  private _timestamp = performance.now();
+  private _timestamp = new Date();
   private _fields = new Map<string, { value: any; type: 'duration' | 'int' }>();
+  private _prefix?: string;
 
   constructor(private _name: string) {}
 
@@ -36,6 +37,24 @@ export class Metric {
     return this._name;
   }
 
+  get prefix() {
+    return this._prefix;
+  }
+
+  get fullName() {
+    return this._prefix ? `${this._prefix}_${this._name}` : this._name;
+  }
+
+  setTimestamp(timestamp: Date) {
+    this._timestamp = timestamp;
+    return this;
+  }
+
+  setPrefix(prefix: string) {
+    this._prefix = prefix;
+    return this;
+  }
+
   addTag(key: string, value: string) {
     this._tags.set(key, value);
     return this;
@@ -49,7 +68,7 @@ export class Metric {
   }
 
   durationField(key: string, duration?: number) {
-    this._fields.set(key, { value: duration ?? performance.now() - this._timestamp, type: 'duration' });
+    this._fields.set(key, { value: duration ?? performance.now() - this._timestamp.getTime(), type: 'duration' });
     return this;
   }
 

backend/src/interfaces/query-metrics.interface.ts

@@ -0,0 +1,6 @@
+export interface IMetricsQueryRepository {
+  queryMaxOverTime(args: QueryMaxOverTimeArgs): Promise<RangeQueryResponse>;
+}
+
+export type RangeQueryResponse = { values: [number, number][] }[];
+export type QueryMaxOverTimeArgs = { metricName: string; start: number; end: number; step: string };

backend/src/monitor.ts

@@ -11,7 +11,7 @@ export function monitorAsyncFunction<T extends AsyncFn>(
   const { name: operationName, tags = {} } = operation;
   const { monitorInvocations = true, acceptedErrors = [] } = options;
 
-  const metric = Metric.create(`${operationPrefix}_${operationName}`);
+  const metric = Metric.create(operationName);
   metric.addTags(tags);
   return async (...args: Parameters<T>) => {
     if (monitorInvocations) {

backend/src/repositories/github.repository.ts

@@ -124,6 +124,9 @@ export class GithubRepository implements IGithubRepository {
         closed: {
           totalCount: number;
         };
+        answered: {
+          totalCount: number;
+        };
       };
     }>(
       `
@@ -138,6 +141,9 @@ export class GithubRepository implements IGithubRepository {
             closed: discussions(states: [CLOSED]) {
               totalCount
             }
+            answered: discussions(answered: true) {
+              totalCount
+            }
           }
         }
       `,
@@ -151,6 +157,7 @@ export class GithubRepository implements IGithubRepository {
       total: repository.total.totalCount,
       open: repository.open.totalCount,
       closed: repository.closed.totalCount,
+      answered: repository.answered.totalCount,
     };
   }
 

backend/src/repositories/influx-metrics-provider.repository.ts

@@ -1,8 +1,8 @@
 import { Point } from '@influxdata/influxdb-client';
-import { IMetricsProviderRepository } from 'src/interfaces/metrics-provider.interface';
+import { IMetricsPushProviderRepository } from 'src/interfaces/metrics-provider.interface';
 import { Metric } from 'src/interfaces/metrics.interface';
 
-export class InfluxMetricsProvider implements IMetricsProviderRepository {
+export class InfluxMetricsPushProvider implements IMetricsPushProviderRepository {
   private metrics: string[] = [];
   constructor(
     private influxApiUrl: string,
@@ -10,7 +10,8 @@ export class InfluxMetricsProvider implements IMetricsProviderRepository {
   ) {}
 
   pushMetric(metric: Metric) {
-    const point = new Point(metric.name);
+    const point = new Point(metric.fullName);
+    point.timestamp(metric.timestamp);
     for (const [key, value] of metric.tags) {
       point.tag(key, value);
     }
@@ -27,19 +28,27 @@ export class InfluxMetricsProvider implements IMetricsProviderRepository {
     }
   }
 
-  async flush() {
+  flush(): Promise<void> {
     if (this.metrics.length === 0) {
-      return;
+      return Promise.resolve();
     }
     const metrics = this.metrics.join('\n');
+    this.metrics = [];
+    const send = async () => {
       const response = await fetch(`${this.influxApiUrl}/write`, {
         method: 'POST',
-      body: this.metrics.join('\n'),
+        body: metrics,
         headers: {
           Authorization: `Token ${this.influxApiToken}`,
+          'Stream-Mode': '1',
         },
       });
       await response.body?.cancel();
-    console.log(metrics);
+      if (response.status !== 204) {
+        throw new Error(`InfluxDB write failed with status ${response.status}`);
+      }
+    };
+
+    return send();
   }
 }

backend/src/repositories/metrics-push.repository.ts

@@ -1,15 +1,15 @@
-import { IMetricsProviderRepository } from 'src/interfaces/metrics-provider.interface';
-import { IMetricsRepository, Metric } from 'src/interfaces/metrics.interface';
+import { IMetricsPushProviderRepository } from 'src/interfaces/metrics-provider.interface';
+import { IMetricsPushRepository, Metric } from 'src/interfaces/metrics.interface';
 import { monitorAsyncFunction } from 'src/monitor';
 import { AsyncFn, Operation, Options } from 'src/types';
 
-export class CloudflareMetricsRepository implements IMetricsRepository {
+export class MetricsPushRepository implements IMetricsPushRepository {
   private readonly defaultTags: { [key: string]: string };
 
   constructor(
-    private operationPrefix: string,
+    private metricPrefix: string,
     tags: Record<string, string>,
-    private metricsProviders: IMetricsProviderRepository[],
+    private metricsProviders: IMetricsPushProviderRepository[],
   ) {
     this.defaultTags = { ...tags };
   }
@@ -26,10 +26,11 @@ export class CloudflareMetricsRepository implements IMetricsRepository {
       }
     };
 
-    return monitorAsyncFunction(this.operationPrefix, operation, call, callback, options);
+    return monitorAsyncFunction(this.metricPrefix, operation, call, callback, options);
   }
 
   push(metric: Metric) {
+    metric.setPrefix(this.metricPrefix);
     metric.addTags(this.defaultTags);
     for (const provider of this.metricsProviders) {
       provider.pushMetric(metric);

backend/src/repositories/metrics-query.repository.ts

@@ -0,0 +1,49 @@
+import {
+  IMetricsQueryRepository,
+  QueryMaxOverTimeArgs,
+  RangeQueryResponse,
+} from 'src/interfaces/query-metrics.interface';
+
+type PrometheusQueryResponse = {
+  data: {
+    result: Array<{
+      metric: object;
+      values: Array<[number, string]>;
+    }>;
+  };
+};
+
+export class MetricsQueryRepository implements IMetricsQueryRepository {
+  constructor(
+    private vmetricsApiUrl: string,
+    private vmetricsApiToken: string,
+    private environment: string,
+  ) {}
+
+  async queryMaxOverTime({ metricName, start, end, step }: QueryMaxOverTimeArgs): Promise<RangeQueryResponse> {
+    const url = new URL(`${this.vmetricsApiUrl}/api/v1/query_range`);
+    const query = `max(max_over_time(${metricName}{environment="${this.environment}", repository_name="immich"}[${step}]))`;
+    const params = {
+      query,
+      start: `${start}`,
+      end: `${end}`,
+      step,
+    };
+    url.search = new URLSearchParams(params).toString();
+    const response = await fetch(url, {
+      method: 'GET',
+      headers: {
+        Authorization: `Bearer ${this.vmetricsApiToken}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    const {
+      data: { result },
+    } = (await new Response(response.body).json()) as PrometheusQueryResponse;
+
+    return result.map((r) => ({
+      values: r.values.map(([timestamp, value]) => [timestamp, parseInt(value)] as [number, number]),
+    }));
+  }
+}

backend/src/workers/api.worker.ts

@@ -1,9 +1,27 @@
-import { IMetricsRepository } from 'src/interfaces/metrics.interface';
+import { DateTime } from 'luxon';
+import { IMetricsQueryRepository } from 'src/interfaces/query-metrics.interface';
 
 export class ApiWorker {
-  constructor(private metricsRepository: IMetricsRepository) {}
+  constructor(private metricsRepository: IMetricsQueryRepository) {}
 
-  getGithubReports() {
-    return [];
+  async getGithubReports() {
+    const metrics = {
+      issues: { metricName: 'immich_data_repository_issue_open_total' },
+      pullRequests: { metricName: 'immich_data_repository_pull_request_open_total' },
+      stars: { metricName: 'immich_data_repository_star_total' },
+      discussions: { metricName: 'immich_data_repository_discussion_total' },
+    };
+
+    const promises = Object.entries(metrics).map(async ([key, value]) => {
+      const results = await this.metricsRepository.queryMaxOverTime({
+        metricName: value.metricName,
+        start: DateTime.now().minus({ years: 1 }).toMillis(),
+        end: DateTime.now().toMillis(),
+        step: '1d',
+      });
+      return [key, results[0]?.values ?? []];
+    });
+
+    return Object.fromEntries(await Promise.all(promises));
   }
 }

backend/src/workers/ingest-processor.worker.ts

@@ -1,11 +1,13 @@
-import { User } from '@octokit/webhooks-types';
-import { GithubRepo } from 'src/constants';
+import { GithubRepo, GithubUser } from 'src/constants';
 import { IGithubRepository } from 'src/interfaces/github.interface';
-import { IMetricsRepository, Metric } from 'src/interfaces/metrics.interface';
+import { IMetricsPushRepository, Metric } from 'src/interfaces/metrics.interface';
 import { QueueItem } from 'src/interfaces/queue.interface';
 
-class GithubMetric extends Metric {
-  withUser(user: User) {
+export class GithubMetric extends Metric {
+  withUser(user?: GithubUser) {
+    if (!user) {
+      return this;
+    }
     return this.addTag('username', user.login).addTag('user_id', user.id.toString());
   }
 
@@ -16,7 +18,7 @@ class GithubMetric extends Metric {
 
 export class IngestProcessorWorker {
   constructor(
-    private metricsRepository: IMetricsRepository,
+    private metricsRepository: IMetricsPushRepository,
     private githubRepository: IGithubRepository,
     private envTag: string,
   ) {}
@@ -28,10 +30,10 @@ export class IngestProcessorWorker {
 
   private async handleMessage(message: Message<QueueItem>) {
     const metrics = {
-      star: new GithubMetric('immich_data_repository_star'),
-      issue: new GithubMetric('immich_data_repository_issue'),
-      pullRequest: new GithubMetric('immich_data_repository_pull_request'),
-      discussion: new GithubMetric('immich_data_repository_discussion'),
+      star: new GithubMetric('star'),
+      issue: new GithubMetric('issue'),
+      pullRequest: new GithubMetric('pull_request'),
+      discussion: new GithubMetric('discussion'),
     };
 
     const { type, data } = message.body;
@@ -71,6 +73,7 @@ export class IngestProcessorWorker {
         .intField('total', discussionCount.total)
         .intField('open_total', discussionCount.open)
         .intField('closed_total', discussionCount.closed)
+        .intField('answered_total', discussionCount.answered)
         .intField('count', count);
     };
 

deployment/.env

@@ -1,7 +1,10 @@
 export TF_VAR_cloudflare_account_id="op://tf/cloudflare/account_id"
 export TF_VAR_cloudflare_api_token="op://tf/cloudflare/api_token"
 export TF_VAR_tf_state_postgres_conn_str="op://tf/tf_state/postgres_conn_str"
-export TF_VAR_vmetrics_api_token="op://tf_$ENVIRONMENT/vmetrics_write_token/token"
+export TF_VAR_vmetrics_monitoring_write_token="op://tf_$ENVIRONMENT/vmetrics_write_token/token"
+export TF_VAR_vmetrics_monitoring_read_token="op://tf_$ENVIRONMENT/vmetrics_read_token/token"
+export TF_VAR_vmetrics_data_write_token="op://tf_$ENVIRONMENT/vmetrics_data_write_token/token"
+export TF_VAR_vmetrics_data_read_token="op://tf_$ENVIRONMENT/vmetrics_data_read_token/token"
 export TF_VAR_github_app_tofu_installation_id="op://tf/github_app_immich_tofu/installation_id"
 export TF_VAR_github_app_tofu_id="op://tf/github_app_immich_tofu/app_id"
 export TF_VAR_github_app_tofu_owner="op://tf/github_app_immich_tofu/owner"

deployment/modules/cloudflare/backend/locals.tf

@@ -8,6 +8,5 @@ locals {
   domain_prefix = "${local.domain_stage}${local.domain_env}"
   domain        = "data.${local.domain_prefix}immich.app"
 
-  monitoring_url   = var.env == "prod" ? "https://cf-workers.monitoring.immich.cloud" : "https://cf-workers.monitoring.dev.immich.cloud"
-  monitoring_token = var.vmetrics_api_token
+  vmetrics_data_api_url = var.env == "prod" ? "https://data.immich.cloud" : "https://data.immich.cloud"
 }

deployment/modules/cloudflare/backend/variables.tf

@@ -1,12 +1,12 @@
 variable "cloudflare_account_id" {}
 variable "tf_state_postgres_conn_str" {}
 variable "dist_dir" {}
-variable "vmetrics_api_token" {}
+variable "vmetrics_data_write_token" {}
+variable "vmetrics_data_read_token" {}
 variable "env" {}
 variable "stage" {
   default = ""
 }
-
 variable "github_app_read_only_id" {}
 variable "github_app_read_only_installation_id" {}
 variable "github_app_read_only_pem_file_pkcs8" {}

deployment/modules/cloudflare/backend/workers.tf

@@ -22,16 +22,21 @@ resource "cloudflare_workers_script" "data_api" {
   }
 
   plain_text_binding {
-    name = "VMETRICS_API_URL"
-    text = local.monitoring_url
+    name = "VMETRICS_DATA_API_URL"
+    text = local.vmetrics_data_api_url
   }
 
   secret_text_binding {
-    name = "VMETRICS_API_TOKEN"
-    text = local.monitoring_token
+    name = "VMETRICS_DATA_WRITE_TOKEN"
+    text = var.vmetrics_data_write_token
   }
 
-  compatibility_date  = "2024-07-29"
+  secret_text_binding {
+    name = "VMETRICS_DATA_READ_TOKEN"
+    text = var.vmetrics_data_read_token
+  }
+
+  compatibility_date  = "2024-12-05"
   compatibility_flags = ["nodejs_compat"]
 }
 
@@ -60,13 +65,13 @@ resource "cloudflare_workers_script" "data_ingest_api" {
   }
 
   plain_text_binding {
-    name = "VMETRICS_API_URL"
-    text = local.monitoring_url
+    name = "VMETRICS_DATA_API_URL"
+    text = local.vmetrics_data_api_url
   }
 
   secret_text_binding {
-    name = "VMETRICS_API_TOKEN"
-    text = local.monitoring_token
+    name = "VMETRICS_DATA_WRITE_TOKEN"
+    text = var.vmetrics_data_write_token
   }
 
   secret_text_binding {
@@ -103,13 +108,13 @@ resource "cloudflare_workers_script" "data_ingest_processor" {
   }
 
   plain_text_binding {
-    name = "VMETRICS_API_URL"
-    text = local.monitoring_url
+    name = "VMETRICS_DATA_API_URL"
+    text = local.vmetrics_data_api_url
   }
 
   secret_text_binding {
-    name = "VMETRICS_API_TOKEN"
-    text = local.monitoring_token
+    name = "VMETRICS_DATA_WRITE_TOKEN"
+    text = var.vmetrics_data_write_token
   }
 
   secret_text_binding {

frontend/package-lock.json

@@ -19,6 +19,7 @@
         "@sveltejs/vite-plugin-svelte": "^5.0.0",
         "@types/eslint": "^9.0.0",
         "@types/luxon": "^3.4.2",
+        "@types/node": "^22.10.2",
         "autoprefixer": "^10.4.19",
         "eslint": "^9.0.0",
         "eslint-config-prettier": "^9.1.0",
@@ -1668,6 +1669,16 @@
         "@types/pbf": "*"
       }
     },
+    "node_modules/@types/node": {
+      "version": "22.10.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz",
+      "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~6.20.0"
+      }
+    },
     "node_modules/@types/parse-json": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@@ -5678,6 +5689,13 @@
         "typescript": ">=4.8.4 <5.8.0"
       }
     },
+    "node_modules/undici-types": {
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/update-browserslist-db": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz",

frontend/package.json

@@ -20,6 +20,7 @@
     "@sveltejs/vite-plugin-svelte": "^5.0.0",
     "@types/eslint": "^9.0.0",
     "@types/luxon": "^3.4.2",
+    "@types/node": "^22.10.2",
     "autoprefixer": "^10.4.19",
     "eslint": "^9.0.0",
     "eslint-config-prettier": "^9.1.0",

frontend/src/lib/components/graphs/github-discussion-graph.svelte

@@ -1,12 +1,16 @@
 <script lang="ts">
   import '$lib/app.css';
-  import starData from '$lib/assets/data/star-data.json';
-  import { VisAxis, VisCrosshair, VisArea, VisTooltip, VisXYContainer } from '@unovis/svelte';
+  import { VisAxis, VisCrosshair, VisLine, VisTooltip, VisXYContainer } from '@unovis/svelte';
   import { DateTime } from 'luxon';
 
   type DataRecord = [DateTime, number];
 
-  const data = starData.map(([timestamp, value]) => [DateTime.fromSeconds(timestamp), value]);
+  type Props = {
+    data: DataRecord[];
+  };
+
+  const { data }: Props = $props();
+
   const x = ([timestamp]: DataRecord) => timestamp.toMillis();
   const y = ([, value]: DataRecord) => value;
   const tickFormatX = (value: number) => DateTime.fromMillis(value).toFormat('MMM yy');
@@ -17,10 +21,10 @@
 </script>
 
 <VisXYContainer {data} height={250} class="area-graph">
-  <VisArea {x} {y} color="royalblue" />
+  <VisLine {x} {y} color="royalblue" />
   <!-- <VisBulletLegend {items} {onLegendItemClick} bulletSize="15px" /> -->
   <VisTooltip />
   <VisCrosshair {x} {y} {template} />
-  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine={false} />
-  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine={true} />
+  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine />
+  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine />
 </VisXYContainer>

frontend/src/lib/components/graphs/github-issue-graph.svelte

@@ -1,14 +1,18 @@
 <script lang="ts">
   import '$lib/app.css';
-  import issueData from '$lib/assets/data/issue-data.json';
-  import { VisAxis, VisCrosshair, VisArea, VisTooltip, VisXYContainer } from '@unovis/svelte';
+  import { VisAxis, VisCrosshair, VisLine, VisTooltip, VisXYContainer } from '@unovis/svelte';
   import { DateTime } from 'luxon';
 
-  type DataRecord = [DateTime, number, number, number, number];
+  type DataRecord = [DateTime, number];
+
+  type Props = {
+    data: DataRecord[];
+  };
+
+  const { data }: Props = $props();
 
-  const data = issueData.map(([timestamp, ...rest]) => [DateTime.fromSeconds(timestamp), ...rest]);
   const x = ([timestamp]: DataRecord) => timestamp.toMillis();
-  const y = ([, , , total]: DataRecord) => total;
+  const y = ([, value]: DataRecord) => value;
   const tickFormatX = (value: number) => DateTime.fromMillis(value).toFormat('MMM yy');
   const tickFormatY = (i: number) =>
     new Intl.NumberFormat(navigator.language, { maximumSignificantDigits: 3 }).format(i);
@@ -17,9 +21,9 @@
 </script>
 
 <VisXYContainer {data} height={250} class="area-graph">
-  <VisArea {x} {y} color="#3fb950" />
+  <VisLine {x} {y} color="#3fb950" />
   <VisTooltip />
   <VisCrosshair {x} {y} {template} />
-  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine={false} />
-  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine={true} />
+  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine />
+  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine />
 </VisXYContainer>

frontend/src/lib/components/graphs/github-pull-request-graph.svelte

@@ -1,18 +1,16 @@
 <script lang="ts">
   import '$lib/app.css';
-  import prData from '$lib/assets/data/pr-data.json';
-  import { VisAxis, VisCrosshair, VisArea, VisTooltip, VisXYContainer } from '@unovis/svelte';
+  import { VisAxis, VisCrosshair, VisLine, VisTooltip, VisXYContainer } from '@unovis/svelte';
   import { DateTime } from 'luxon';
 
-  type DataRecord = [DateTime, number, number, number];
+  type DataRecord = [DateTime, number];
 
-  // remove headers
-  prData.shift();
+  type Props = {
+    data: DataRecord[];
+  };
+
+  const { data }: Props = $props();
 
-  const data = (prData as Array<[number, number, number, number]>).map(([timestamp, , , total]) => [
-    DateTime.fromSeconds(timestamp),
-    total,
-  ]);
   const x = ([timestamp]: DataRecord) => timestamp.toMillis();
   const y = ([, value]: DataRecord) => value;
   const tickFormatX = (value: number) => DateTime.fromMillis(value).toFormat('MMM yy');
@@ -23,9 +21,9 @@
 </script>
 
 <VisXYContainer {data} height={250} class="area-graph">
-  <VisArea {x} {y} color="#ab7df8" />
+  <VisLine {x} {y} color="#ab7df8" />
   <VisTooltip />
   <VisCrosshair {x} {y} {template} />
-  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine={false} />
-  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine={true} />
+  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine />
+  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine />
 </VisXYContainer>

frontend/src/lib/components/graphs/github-stars-graph.svelte

@@ -1,12 +1,16 @@
 <script lang="ts">
   import '$lib/app.css';
-  import starData from '$lib/assets/data/star-data.json';
-  import { VisAxis, VisCrosshair, VisArea, VisTooltip, VisXYContainer } from '@unovis/svelte';
+  import { VisAxis, VisCrosshair, VisLine, VisTooltip, VisXYContainer } from '@unovis/svelte';
   import { DateTime } from 'luxon';
 
   type DataRecord = [DateTime, number];
 
-  const data = starData.map(([timestamp, value]) => [DateTime.fromSeconds(timestamp), value]);
+  type Props = {
+    data: DataRecord[];
+  };
+
+  const { data }: Props = $props();
+
   const x = ([timestamp]: DataRecord) => timestamp.toMillis();
   const y = ([, value]: DataRecord) => value;
   const tickFormatX = (value: number) => DateTime.fromMillis(value).toFormat('MMM yy');
@@ -17,9 +21,9 @@
 </script>
 
 <VisXYContainer {data} height={250} class="area-graph">
-  <VisArea {x} {y} color="#ffc501" />
+  <VisLine {x} {y} color="#ffc501" />
   <VisTooltip />
   <VisCrosshair {x} {y} {template} />
-  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine={false} />
-  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine={true} />
+  <VisAxis tickFormat={tickFormatX} type="x" numTicks={6} gridLine />
+  <VisAxis tickFormat={tickFormatY} type="y" numTicks={4} gridLine />
 </VisXYContainer>

frontend/src/lib/services/api.svelte.ts

@@ -0,0 +1,39 @@
+import { DateTime } from 'luxon';
+
+type GithubDataResponse = {
+  issues: GithubReportData;
+  pullRequests: GithubReportData;
+  stars: GithubReportData;
+  discussions: GithubReportData;
+};
+type GithubReportData = Array<[number, number]>;
+type GithubDataRecord = [DateTime, number];
+
+type GithubData = {
+  issues: GithubDataRecord[];
+  pullRequests: GithubDataRecord[];
+  stars: GithubDataRecord[];
+  discussions: GithubDataRecord[];
+};
+
+export const githubData = $state<GithubData>({
+  stars: [],
+  issues: [],
+  pullRequests: [],
+  discussions: [],
+});
+
+const asTimestamp = ([timestamp, value]: [number, number]) =>
+  [DateTime.fromSeconds(timestamp), value] as [DateTime, number];
+
+export const loadGithubData = async () => {
+  const response = await fetch('/api/github');
+  if (response.ok) {
+    const { stars, issues, pullRequests, discussions } = (await response.json()) as GithubDataResponse;
+
+    githubData.stars = stars.map(asTimestamp);
+    githubData.pullRequests = pullRequests.map(asTimestamp);
+    githubData.issues = issues.map(asTimestamp);
+    githubData.discussions = discussions.map(asTimestamp);
+  }
+};

frontend/src/routes/+page.svelte

@@ -5,6 +5,7 @@
   import GithubPullRequestGraph from '$lib/components/graphs/github-pull-request-graph.svelte';
   import GithubStarsGraph from '$lib/components/graphs/github-stars-graph.svelte';
   import Icon from '$lib/components/icon.svelte';
+  import { githubData } from '$lib/services/api.svelte';
   import { mdiBug, mdiGithub, mdiMessageOutline, mdiSourceBranch, mdiStarOutline } from '@mdi/js';
 </script>
 
@@ -20,7 +21,7 @@
     <section class="flex flex-col gap-4">
       <h2 class="flex gap-1 text-xl">
         <Icon path={mdiGithub} size="24" />
-        <span>Repro Trends</span>
+        <span>Repo Trends</span>
       </h2>
       <div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
         <div>
@@ -28,28 +29,28 @@
             <Icon path={mdiStarOutline} size="24" />
             <span>Stars</span>
           </h3>
-          <GithubStarsGraph />
+          <GithubStarsGraph data={githubData.stars} />
         </div>
         <div>
           <h3 class="flex gap-1 text-lg">
             <Icon path={mdiBug} size="24" />
             <span>Open Issues</span>
           </h3>
-          <GithubIssueGraph />
+          <GithubIssueGraph data={githubData.issues} />
         </div>
         <div>
           <h3 class="flex gap-1 text-lg">
             <Icon path={mdiSourceBranch} size="24" />
             <span>Open Pull Requests</span>
           </h3>
-          <GithubPullRequestGraph />
+          <GithubPullRequestGraph data={githubData.pullRequests} />
         </div>
         <div>
           <h3 class="flex gap-1 text-lg">
             <Icon path={mdiMessageOutline} size="24" />
             <span>Total Discussions</span>
           </h3>
-          <GithubDiscussionGraph />
+          <GithubDiscussionGraph data={githubData.discussions} />
         </div>
       </div>
     </section>

frontend/src/routes/+page.ts

@@ -1 +1,9 @@
+import { loadGithubData } from '$lib/services/api.svelte';
+
 export const ssr = false;
+
+export const load = async () => {
+  await loadGithubData();
+
+  return {};
+};

frontend/tsconfig.json

@@ -11,9 +11,4 @@
     "strict": true,
     "moduleResolution": "bundler"
   }
-  // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
-  // except $lib which is handled by https://kit.svelte.dev/docs/configuration#files
-  //
-  // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
-  // from the referenced tsconfig.json - TypeScript does not merge them in
 }

frontend/vite.config.ts

@@ -1,6 +1,20 @@
 import { sveltekit } from '@sveltejs/kit/vite';
 import { defineConfig } from 'vite';
 
+const apiHost = process.env.IMMICH_BACKEND || 'https://data.immich.app/';
+
+console.log(`Connecting to ${apiHost}`);
+
 export default defineConfig({
   plugins: [sveltekit()],
+  server: {
+    proxy: {
+      '/api': {
+        target: apiHost,
+        secure: true,
+        changeOrigin: true,
+        ws: true,
+      },
+    },
+  },
 });