diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml index f7b10f6..61dab91 100644 --- a/.idea/watcherTasks.xml +++ b/.idea/watcherTasks.xml @@ -1,7 +1,7 @@ - + - \ No newline at end of file + diff --git a/backend/package.json b/backend/package.json index 2294a0c..831e984 100644 --- a/backend/package.json +++ b/backend/package.json @@ -15,7 +15,8 @@ "lint:fix": "npm run lint -- --fix", "format": "prettier --check .", "format:fix": "prettier --write .", - "check": "tsc --noEmit" + "check": "tsc --noEmit", + "backfill:reddit": "tsx src/bin/backfill-reddit.ts" }, "devDependencies": { "@cloudflare/vitest-pool-workers": "^0.8.0", diff --git a/backend/src/bin/backfill-reddit.ts b/backend/src/bin/backfill-reddit.ts new file mode 100644 index 0000000..e09d477 --- /dev/null +++ b/backend/src/bin/backfill-reddit.ts @@ -0,0 +1,237 @@ +import { DateTime } from 'luxon'; +import { Metric } from 'src/interfaces/metrics.interface'; +import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provider.repository'; +import { MetricsPushRepository } from 'src/repositories/metrics-push.repository'; + +interface RedditDataPoint { + timestamp: DateTime; + subscribers: number; +} + +// Historical data points we know about +const KNOWN_DATA_POINTS: RedditDataPoint[] = [ + { timestamp: DateTime.fromSeconds(1649635200), subscribers: 2 }, + { timestamp: DateTime.fromSeconds(1660262400), subscribers: 21 }, + { timestamp: DateTime.fromSeconds(1661040000), subscribers: 26 }, + { timestamp: DateTime.fromSeconds(1661126400), subscribers: 27 }, + { timestamp: DateTime.fromSeconds(1662076800), subscribers: 38 }, + { timestamp: DateTime.fromSeconds(1665360000), subscribers: 59 }, + { timestamp: DateTime.fromSeconds(1667952000), subscribers: 87 }, + { timestamp: DateTime.fromSeconds(1669680000), subscribers: 111 }, + { timestamp: DateTime.fromSeconds(1671321600), subscribers: 124 }, + { timestamp: DateTime.fromSeconds(1672358400), subscribers: 137 }, + { timestamp: DateTime.fromSeconds(1673308800), subscribers: 160 }, + { timestamp: DateTime.fromSeconds(1674259200), subscribers: 180 }, + { timestamp: DateTime.fromSeconds(1676332800), subscribers: 210 }, + { timestamp: DateTime.fromSeconds(1677801600), subscribers: 236 }, + { timestamp: DateTime.fromSeconds(1678579200), subscribers: 246 }, + { timestamp: DateTime.fromSeconds(1679270400), subscribers: 254 }, + { timestamp: DateTime.fromSeconds(1680307200), subscribers: 281 }, + { timestamp: DateTime.fromSeconds(1680652800), subscribers: 299 }, + { timestamp: DateTime.fromSeconds(1681084800), subscribers: 319 }, + { timestamp: DateTime.fromSeconds(1681603200), subscribers: 335 }, + { timestamp: DateTime.fromSeconds(1682035200), subscribers: 354 }, + { timestamp: DateTime.fromSeconds(1682380800), subscribers: 368 }, + { timestamp: DateTime.fromSeconds(1682985600), subscribers: 389 }, + { timestamp: DateTime.fromSeconds(1683590400), subscribers: 410 }, + { timestamp: DateTime.fromSeconds(1684022400), subscribers: 424 }, + { timestamp: DateTime.fromSeconds(1684454400), subscribers: 444 }, + { timestamp: DateTime.fromSeconds(1684800000), subscribers: 474 }, + { timestamp: DateTime.fromSeconds(1685318400), subscribers: 500 }, + { timestamp: DateTime.fromSeconds(1685750400), subscribers: 539 }, + { timestamp: DateTime.fromSeconds(1686096000), subscribers: 567 }, + { timestamp: DateTime.fromSeconds(1686441600), subscribers: 585 }, + { timestamp: DateTime.fromSeconds(1687046400), subscribers: 590 }, + { timestamp: DateTime.fromSeconds(1687478400), subscribers: 615 }, + { timestamp: DateTime.fromSeconds(1687824000), subscribers: 643 }, + { timestamp: DateTime.fromSeconds(1688256000), subscribers: 670 }, + { timestamp: DateTime.fromSeconds(1688601600), subscribers: 695 }, + { timestamp: DateTime.fromSeconds(1688947200), subscribers: 722 }, + { timestamp: DateTime.fromSeconds(1689292800), subscribers: 748 }, + { timestamp: DateTime.fromSeconds(1689811200), subscribers: 782 }, + { timestamp: DateTime.fromSeconds(1690502400), subscribers: 836 }, + { timestamp: DateTime.fromSeconds(1690761600), subscribers: 871 }, + { timestamp: DateTime.fromSeconds(1691280000), subscribers: 908 }, + { timestamp: DateTime.fromSeconds(1691625600), subscribers: 927 }, + { timestamp: DateTime.fromSeconds(1692403200), subscribers: 978 }, + { timestamp: DateTime.fromSeconds(1692662400), subscribers: 1013 }, + { timestamp: DateTime.fromSeconds(1693267200), subscribers: 1056 }, + { timestamp: DateTime.fromSeconds(1693699200), subscribers: 1095 }, + { timestamp: DateTime.fromSeconds(1693958400), subscribers: 1118 }, + { timestamp: DateTime.fromSeconds(1694304000), subscribers: 1133 }, + { timestamp: DateTime.fromSeconds(1694649600), subscribers: 1153 }, + { timestamp: DateTime.fromSeconds(1694995200), subscribers: 1174 }, + { timestamp: DateTime.fromSeconds(1695340800), subscribers: 1217 }, + { timestamp: DateTime.fromSeconds(1695600000), subscribers: 1243 }, + { timestamp: DateTime.fromSeconds(1695945600), subscribers: 1281 }, + { timestamp: DateTime.fromSeconds(1696204800), subscribers: 1302 }, + { timestamp: DateTime.fromSeconds(1696550400), subscribers: 1355 }, + { timestamp: DateTime.fromSeconds(1696809600), subscribers: 1401 }, + { timestamp: DateTime.fromSeconds(1697241600), subscribers: 1453 }, + { timestamp: DateTime.fromSeconds(1697500800), subscribers: 1483 }, + { timestamp: DateTime.fromSeconds(1697760000), subscribers: 1503 }, + { timestamp: DateTime.fromSeconds(1698019200), subscribers: 1545 }, + { timestamp: DateTime.fromSeconds(1698451200), subscribers: 1597 }, + { timestamp: DateTime.fromSeconds(1698710400), subscribers: 1641 }, + { timestamp: DateTime.fromSeconds(1698969600), subscribers: 1697 }, + { timestamp: DateTime.fromSeconds(1699315200), subscribers: 1752 }, + { timestamp: DateTime.fromSeconds(1699660800), subscribers: 1788 }, + { timestamp: DateTime.fromSeconds(1699833600), subscribers: 1812 }, + { timestamp: DateTime.fromSeconds(1700092800), subscribers: 1875 }, + { timestamp: DateTime.fromSeconds(1700438400), subscribers: 1949 }, + { timestamp: DateTime.fromSeconds(1700784000), subscribers: 2016 }, + { timestamp: DateTime.fromSeconds(1701129600), subscribers: 2068 }, + { timestamp: DateTime.fromSeconds(1701475200), subscribers: 2136 }, + { timestamp: DateTime.fromSeconds(1701820800), subscribers: 2192 }, + { timestamp: DateTime.fromSeconds(1702166400), subscribers: 2251 }, + { timestamp: DateTime.fromSeconds(1702512000), subscribers: 2310 }, + { timestamp: DateTime.fromSeconds(1702684800), subscribers: 2353 }, + { timestamp: DateTime.fromSeconds(1703548800), subscribers: 2542 }, + { timestamp: DateTime.fromSeconds(1704575992), subscribers: 2826 }, + // Gap in historical data + { timestamp: DateTime.fromSeconds(1709528400), subscribers: 4000 }, + { timestamp: DateTime.fromSeconds(1712203200), subscribers: 5000 }, + { timestamp: DateTime.fromSeconds(1724403398), subscribers: 9476 }, + { timestamp: DateTime.fromSeconds(1727411726), subscribers: 10884 }, + { timestamp: DateTime.fromSeconds(1733279612), subscribers: 14362 }, + { timestamp: DateTime.fromSeconds(1734757200), subscribers: 16000 }, + { timestamp: DateTime.fromSeconds(1734878583), subscribers: 16167 }, + { timestamp: DateTime.fromSeconds(1735490966), subscribers: 17012 }, + { timestamp: DateTime.fromSeconds(1738040400), subscribers: 20000 }, + { timestamp: DateTime.fromSeconds(1739125889), subscribers: 20767 }, + { timestamp: DateTime.fromSeconds(1741468187), subscribers: 22532 }, + { timestamp: DateTime.fromSeconds(1741474683), subscribers: 22604 }, + { timestamp: DateTime.fromSeconds(1741826945), subscribers: 22829 }, + { timestamp: DateTime.fromSeconds(1743096105), subscribers: 23771 }, + { timestamp: DateTime.fromSeconds(1743900139), subscribers: 24353 }, + { timestamp: DateTime.fromSeconds(1745763306), subscribers: 25611 }, + { timestamp: DateTime.fromSeconds(1747627200), subscribers: 27000 }, + { timestamp: DateTime.fromSeconds(1748727093), subscribers: 27901 }, + { timestamp: DateTime.fromSeconds(1749960000), subscribers: 28756 }, + { timestamp: DateTime.fromSeconds(1752379200), subscribers: 30378 }, + { timestamp: DateTime.fromSeconds(1752653774), subscribers: 30480 }, + { timestamp: DateTime.fromSeconds(1753142400), subscribers: 30799 }, + { timestamp: DateTime.fromSeconds(1753574400), subscribers: 31199 }, + { timestamp: DateTime.fromSeconds(1753920000), subscribers: 31469 }, + { timestamp: DateTime.fromSeconds(1754352000), subscribers: 31776 }, + { timestamp: DateTime.fromSeconds(1754625600), subscribers: 32000 }, + { timestamp: DateTime.fromSeconds(1755216000), subscribers: 32547 }, + { timestamp: DateTime.fromSeconds(1755475200), subscribers: 32775 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 19)), subscribers: 32825 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 20)), subscribers: 32947 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 21)), subscribers: 33042 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 22)), subscribers: 33108 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 23)), subscribers: 33219 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 24)), subscribers: 33321 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 25)), subscribers: 33390 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 26)), subscribers: 33495 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 27)), subscribers: 33597 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 28)), subscribers: 33665 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 29)), subscribers: 33729 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 30)), subscribers: 33813 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 7, 31)), subscribers: 33889 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 1)), subscribers: 33977 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 2)), subscribers: 34105 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 3)), subscribers: 34204 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 4)), subscribers: 34289 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 5)), subscribers: 34369 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 6)), subscribers: 34426 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 7)), subscribers: 34482 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 8)), subscribers: 34554 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 9)), subscribers: 34648 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 10)), subscribers: 34712 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 11)), subscribers: 34792 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 12)), subscribers: 34870 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 13)), subscribers: 34954 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 14)), subscribers: 35061 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 15)), subscribers: 35182 }, + { timestamp: DateTime.fromJSDate(new Date(2025, 8, 16)), subscribers: 35326 }, +]; + +class RedditBackfillService { + constructor( + private metricsRepository: MetricsPushRepository, + private influxProvider: InfluxMetricsPushProvider, + private env: string, + ) {} + + async backfillReddit(subreddit: string = 'immich') { + console.log(`Starting Reddit backfill for r/${subreddit}`); + + // Delete existing time series + await this.deleteTimeSeries('immich_data_repository_reddit_subscriber_.*', this.env); + + const sortedData = KNOWN_DATA_POINTS.sort((a, b) => a.timestamp.toMillis() - b.timestamp.toMillis()); + console.log(`Processing ${sortedData.length} hardcoded data points`); + + // Push metrics to InfluxDB + let count = 0; + for (const dataPoint of sortedData) { + count++; + const metric = new Metric('reddit_subscriber') + .setTimestamp(dataPoint.timestamp.toJSDate()) + .intField('total', dataPoint.subscribers) + .addTag('environment', this.env); + + this.metricsRepository.push(metric); + + // Flush every 100 metrics + if (count % 100 === 0) { + await this.influxProvider.flush(); + console.log(`Flushed ${count} metrics...`); + } + } + + await this.influxProvider.flush(); + console.log(`Reddit backfill complete. Total metrics pushed: ${count}`); + } + + private async deleteTimeSeries(namePattern: string, environment: string) { + const url = new URL(`${process.env.VMETRICS_DATA_API_URL}/api/v1/admin/tsdb/delete_series`); + const match = `{__name__=~'${namePattern}',environment='${environment}'}`; + url.search = new URLSearchParams({ ['match[]']: match }).toString(); + + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${process.env.VMETRICS_DATA_ADMIN_TOKEN}`, + 'Content-Type': 'application/json', + }, + }); + + console.log(`Deleting time series ${match}`); + if (response.status !== 204) { + console.log(response.status); + throw new Error(`Failed to delete time series ${match}`); + } + } +} + +// Main execution +const main = async () => { + if ( + !process.env.VMETRICS_DATA_WRITE_TOKEN || + !process.env.VMETRICS_DATA_API_URL || + !process.env.VMETRICS_DATA_ADMIN_TOKEN + ) { + throw new Error('Missing VMetrics API credentials'); + } + + if (!process.env.ENVIRONMENT) { + throw new Error('Missing environment'); + } + + const env = `${process.env.ENVIRONMENT}${process.env.TF_VAR_stage ? `_${process.env.TF_VAR_stage}` : ''}`; + const influxProvider = new InfluxMetricsPushProvider( + process.env.VMETRICS_DATA_API_URL, + process.env.VMETRICS_DATA_WRITE_TOKEN, + ); + const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]); + + const service = new RedditBackfillService(metricsRepository, influxProvider, env); + await service.backfillReddit('immich'); +}; + +main() + .then(() => console.log('Reddit backfill complete')) + .catch(console.error); diff --git a/backend/src/index.ts b/backend/src/index.ts index 22efb9d..edf4c21 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,6 +9,7 @@ import { MetricsQueryRepository } from 'src/repositories/metrics-query.repositor import { ApiWorker } from 'src/workers/api.worker'; import { IngestApiWorker } from 'src/workers/ingest-api.worker'; import { IngestProcessorWorker } from 'src/workers/ingest-processor.worker'; +import { RedditIngestWorker } from 'src/workers/reddit-ingest.worker'; type FetchRequest = IRequest & Parameters[0]; @@ -28,7 +29,7 @@ const newIngestApiWorker = (request: FetchRequest, env: WorkerEnv) => { return new IngestApiWorker(queue); }; -const withSlug = (req: FetchRequest, env: WorkerEnv) => { +const slugAuth = (req: FetchRequest, env: WorkerEnv) => { const slug = env.SLUG; if (slug && req.params.slug !== slug) { return error(401, 'Unauthorized'); @@ -40,8 +41,9 @@ const handleError = (err: Error) => { }; const router = AutoRouter() - .get('/api/github', (...args) => newApiWorker(...args).getGithubReports()) - .post('/ingest/github/:slug', withSlug, async (req, env) => + .get('/api/github', (...args) => newApiWorker(...args).getGithubData()) + .get('/api/reddit', (...args) => newApiWorker(...args).getRedditData()) + .post('/ingest/github/:slug', slugAuth, async (req, env) => newIngestApiWorker(req, env) .onGithubEvent(await req.json()) .catch(handleError) @@ -66,4 +68,16 @@ export default { await influxProvider.flush(); }, + scheduled: async (event, env) => { + const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN); + const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]); + const redditWorker = new RedditIngestWorker(metricsRepository, asEnvTag(env)); + + try { + await redditWorker.fetchAndStoreCurrentMetrics('immich'); + await influxProvider.flush(); + } catch (error) { + console.error('Failed to fetch Reddit metrics:', error); + } + }, } satisfies ExportedHandler; diff --git a/backend/src/interfaces/query-metrics.interface.ts b/backend/src/interfaces/query-metrics.interface.ts index 3ae0de9..4835d0e 100644 --- a/backend/src/interfaces/query-metrics.interface.ts +++ b/backend/src/interfaces/query-metrics.interface.ts @@ -3,4 +3,4 @@ export interface IMetricsQueryRepository { } export type RangeQueryResponse = { values: [number, number][] }[]; -export type QueryMaxOverTimeArgs = { metricName: string; start: number; end: number; step: string }; +export type QueryMaxOverTimeArgs = { metricName: string; start: number; end: number; step: string; labels: string[] }; diff --git a/backend/src/repositories/metrics-query.repository.ts b/backend/src/repositories/metrics-query.repository.ts index 66c76c7..cb76055 100644 --- a/backend/src/repositories/metrics-query.repository.ts +++ b/backend/src/repositories/metrics-query.repository.ts @@ -20,9 +20,9 @@ export class MetricsQueryRepository implements IMetricsQueryRepository { private environment: string, ) {} - async queryMaxOverTime({ metricName, start, end, step }: QueryMaxOverTimeArgs): Promise { + async queryMaxOverTime({ metricName, start, end, step, labels }: QueryMaxOverTimeArgs): Promise { const url = new URL(`${this.vmetricsApiUrl}/api/v1/query_range`); - const query = `max(max_over_time(${metricName}{environment="${this.environment}", repository_name="immich"}[${step}]))`; + const query = `interpolate(max(max_over_time(${metricName}{environment="${this.environment}"${labels.length > 0 ? ',' + labels.join(',') : ''}}[${step}])))`; const params = { query, start: `${start}`, diff --git a/backend/src/repositories/reddit.repository.ts b/backend/src/repositories/reddit.repository.ts new file mode 100644 index 0000000..08d1e39 --- /dev/null +++ b/backend/src/repositories/reddit.repository.ts @@ -0,0 +1,35 @@ +import fetchRetry from 'fetch-retry'; + +const fetch = fetchRetry(globalThis.fetch, { + retries: 3, + retryDelay: 1000, +}); + +export interface RedditSubredditData { + subscribers: number; +} + +export interface RedditAboutResponse { + data: RedditSubredditData; +} + +export class RedditRepository { + private readonly userAgent = 'Mozilla/5.0 (compatible; ImmichDataBot/1.0)'; + + async getSubredditData(subreddit: string): Promise { + const url = `https://www.reddit.com/r/${subreddit}/about.json`; + + const response = await fetch(url, { + headers: { + 'User-Agent': this.userAgent, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch Reddit data: ${response.status} ${response.statusText}`); + } + + const data = (await response.json()) as RedditAboutResponse; + return data.data; + } +} diff --git a/backend/src/workers/api.worker.ts b/backend/src/workers/api.worker.ts index 171c766..78a1238 100644 --- a/backend/src/workers/api.worker.ts +++ b/backend/src/workers/api.worker.ts @@ -5,21 +5,31 @@ import { IMetricsQueryRepository } from 'src/interfaces/query-metrics.interface' export class ApiWorker { constructor(private metricsRepository: IMetricsQueryRepository) {} - async getGithubReports() { - const metrics = { - issues: { metricName: 'immich_data_repository_issue_open_total' }, - pullRequests: { metricName: 'immich_data_repository_pull_request_open_total' }, - stars: { metricName: 'immich_data_repository_star_total' }, - discussions: { metricName: 'immich_data_repository_discussion_total' }, - mergedPullRequests: { metricName: 'immich_data_repository_pull_request_merged_total' }, - }; + async getGithubData() { + const labels = ['repository_name="immich"']; + return this.asTimeSeries({ + issues: { metricName: 'immich_data_repository_issue_open_total', labels }, + pullRequests: { metricName: 'immich_data_repository_pull_request_open_total', labels }, + stars: { metricName: 'immich_data_repository_star_total', labels }, + discussions: { metricName: 'immich_data_repository_discussion_total', labels }, + mergedPullRequests: { metricName: 'immich_data_repository_pull_request_merged_total', labels }, + }); + } + async getRedditData() { + return this.asTimeSeries({ + subscribers: { metricName: 'immich_data_repository_reddit_subscriber_total', labels: [] }, + }); + } + + async asTimeSeries(metrics: Record) { const promises = Object.entries(metrics).map(async ([key, value]) => { const results = await this.metricsRepository.queryMaxOverTime({ metricName: value.metricName, start: IMMICH_INCEPTION.toMillis(), end: DateTime.now().toMillis(), step: '1d', + labels: value.labels, }); return [key, results[0]?.values ?? []]; }); diff --git a/backend/src/workers/reddit-ingest.worker.ts b/backend/src/workers/reddit-ingest.worker.ts new file mode 100644 index 0000000..965980a --- /dev/null +++ b/backend/src/workers/reddit-ingest.worker.ts @@ -0,0 +1,28 @@ +import { IMetricsPushRepository, Metric } from 'src/interfaces/metrics.interface'; +import { RedditRepository } from 'src/repositories/reddit.repository'; + +export class RedditIngestWorker { + private redditRepository = new RedditRepository(); + + constructor( + private metricsRepository: IMetricsPushRepository, + private envTag: string, + ) {} + + async fetchAndStoreCurrentMetrics(subreddit: string = 'immich') { + try { + const data = await this.redditRepository.getSubredditData(subreddit); + + const metric = new Metric('reddit_subscriber') + .intField('total', data.subscribers) + .addTag('environment', this.envTag); + + this.metricsRepository.push(metric); + + return data.subscribers; + } catch (error) { + console.error(`Failed to fetch Reddit data for r/${subreddit}:`, error); + throw error; + } + } +} diff --git a/backend/wrangler.toml b/backend/wrangler.toml index bca6a74..a33428c 100644 --- a/backend/wrangler.toml +++ b/backend/wrangler.toml @@ -21,3 +21,6 @@ SLUG = "slug" max_batch_timeout = 30 max_retries = 10 dead_letter_queue = "data-ingest-dev-dlq" + +[triggers] +crons = ["0 */6 * * *"] diff --git a/deployment/modules/cloudflare/backend/workers.tf b/deployment/modules/cloudflare/backend/workers.tf index 9fa34fd..49ca1ac 100644 --- a/deployment/modules/cloudflare/backend/workers.tf +++ b/deployment/modules/cloudflare/backend/workers.tf @@ -136,6 +136,45 @@ resource "cloudflare_workers_script" "data_ingest_processor" { compatibility_flags = ["nodejs_compat"] } +resource "cloudflare_workers_script" "data_ingest_cron" { + account_id = var.cloudflare_account_id + name = "data-ingest-cron${local.resource_suffix}" + content = file("${var.dist_dir}/backend/index.js") + module = true + + plain_text_binding { + name = "ENVIRONMENT" + text = var.env + } + + dynamic "plain_text_binding" { + for_each = var.stage != "" ? [var.stage] : [] + content { + name = "STAGE" + text = var.stage + } + } + + plain_text_binding { + name = "VMETRICS_DATA_API_URL" + text = local.vmetrics_data_api_url + } + + secret_text_binding { + name = "VMETRICS_DATA_WRITE_TOKEN" + text = var.vmetrics_data_write_token + } + + compatibility_date = "2025-09-17" + compatibility_flags = ["nodejs_compat"] +} + +resource "cloudflare_workers_cron_trigger" "data_ingest_cron" { + account_id = var.cloudflare_account_id + script_name = cloudflare_workers_script.data_ingest_cron.name + schedules = [ "*/5 * * * *" ] +} + locals { data_api_url = "${local.domain}/api" ingest_api_url = "${local.domain}/ingest" diff --git a/frontend/src/lib/components/graphs/github-graph.svelte b/frontend/src/lib/components/graphs/github-graph.svelte index 5460133..40f1337 100644 --- a/frontend/src/lib/components/graphs/github-graph.svelte +++ b/frontend/src/lib/components/graphs/github-graph.svelte @@ -5,7 +5,7 @@ import uPlot, { type Axis } from 'uplot'; type DataRecord = [number, number]; - type Colors = 'yellow' | 'green' | 'purple' | 'blue'; + type Colors = 'yellow' | 'green' | 'purple' | 'blue' | 'orange'; type Props = { id: string; @@ -32,6 +32,7 @@ green: 'rgb(70, 183, 83)', purple: 'rgb(172, 124, 249)', blue: 'rgb(63, 106, 222)', + orange: 'rgb(255, 69, 0)', }; let areaColor = { @@ -39,6 +40,7 @@ green: 'rgba(70, 183, 83, 0.1)', purple: 'rgba(172, 124, 249, 0.1)', blue: 'rgba(63, 106, 222, 0.1)', + orange: 'rgba(255, 69, 0, 0.1)', }; const axis: Axis = { diff --git a/frontend/src/lib/services/api.svelte.ts b/frontend/src/lib/services/api.svelte.ts index 013faed..caa596d 100644 --- a/frontend/src/lib/services/api.svelte.ts +++ b/frontend/src/lib/services/api.svelte.ts @@ -1,18 +1,18 @@ type GithubDataResponse = { - issues: GithubReportData; - pullRequests: GithubReportData; - stars: GithubReportData; - discussions: GithubReportData; - mergedPullRequests: GithubReportData; + issues: ReportData; + pullRequests: ReportData; + stars: ReportData; + discussions: ReportData; + mergedPullRequests: ReportData; }; -type GithubReportData = Array<[number, number]>; -type GithubDataRecord = [number, number]; +type ReportData = Array<[number, number]>; +type DataRecord = [number, number]; -export const githubStars = $state<{ value?: GithubDataRecord[] }>({}); -export const githubIssues = $state<{ value?: GithubDataRecord[] }>({}); -export const githubPullRequests = $state<{ value?: GithubDataRecord[] }>({}); -export const githubDiscussions = $state<{ value?: GithubDataRecord[] }>({}); -export const githubMergedPullRequests = $state<{ value?: GithubDataRecord[] }>({}); +export const githubStars = $state<{ value?: DataRecord[] }>({}); +export const githubIssues = $state<{ value?: DataRecord[] }>({}); +export const githubPullRequests = $state<{ value?: DataRecord[] }>({}); +export const githubDiscussions = $state<{ value?: DataRecord[] }>({}); +export const githubMergedPullRequests = $state<{ value?: DataRecord[] }>({}); export const loadGithubData = async () => { const response = await fetch('/api/github'); @@ -27,3 +27,17 @@ export const loadGithubData = async () => { githubMergedPullRequests.value = mergedPullRequests; } }; + +type RedditDataResponse = { + subscribers: ReportData; +}; + +export const redditMembers = $state<{ value?: DataRecord[] }>({}); + +export const loadRedditData = async () => { + const response = await fetch('/api/reddit'); + if (response.ok) { + const { subscribers } = (await response.json()) as RedditDataResponse; + redditMembers.value = subscribers; + } +}; diff --git a/frontend/src/routes/+page.svelte b/frontend/src/routes/+page.svelte index ea4f324..1dd07b8 100644 --- a/frontend/src/routes/+page.svelte +++ b/frontend/src/routes/+page.svelte @@ -8,6 +8,8 @@ githubStars, githubMergedPullRequests, loadGithubData, + loadRedditData, + redditMembers, } from '$lib/services/api.svelte'; import { Card, @@ -22,7 +24,7 @@ Stack, Text, } from '@immich/ui'; - import { mdiBugOutline, mdiMessageOutline, mdiSourceBranch, mdiStarOutline } from '@mdi/js'; + import { mdiAccountGroup, mdiBugOutline, mdiMessageOutline, mdiSourceBranch, mdiStarOutline } from '@mdi/js'; import { DateTime } from 'luxon'; import uPlot from 'uplot'; import 'uplot/dist/uPlot.min.css'; @@ -42,6 +44,7 @@ const futo = DateTime.fromObject({ year: 2024, month: 5, day: 1 }, { zone: 'America/Chicago' }); void loadGithubData(); + void loadRedditData(); @@ -49,7 +52,7 @@ Immich Data
- Statistics + Statistics
@@ -101,9 +104,7 @@
- - GitHub Trends - + GitHub immich-app/immich
@@ -192,5 +193,33 @@
+ +
+
+ Reddit + r/immich +
+
+ + + + + + Total Members + + + + + + + +
+