From ffb8e717dfece5e2e7e652ae6f269967dd41b645 Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Thu, 18 Sep 2025 20:28:48 +0100 Subject: [PATCH] feat: discord stats (#188) --- backend/package.json | 3 +- backend/src/bin/backfill-discord.ts | 192 ++++++++++++++++++ backend/src/bin/backfill.ts | 1 - backend/src/index.ts | 12 +- .../src/repositories/discord.repository.ts | 118 +++++++++++ backend/src/workers/api.worker.ts | 6 + backend/src/workers/discord-ingest.worker.ts | 44 ++++ frontend/src/lib/services/api.svelte.ts | 22 ++ frontend/src/routes/+page.svelte | 28 ++- 9 files changed, 418 insertions(+), 8 deletions(-) create mode 100644 backend/src/bin/backfill-discord.ts create mode 100644 backend/src/repositories/discord.repository.ts create mode 100644 backend/src/workers/discord-ingest.worker.ts diff --git a/backend/package.json b/backend/package.json index 831e984..ae65108 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,7 +16,8 @@ "format": "prettier --check .", "format:fix": "prettier --write .", "check": "tsc --noEmit", - "backfill:reddit": "tsx src/bin/backfill-reddit.ts" + "backfill:reddit": "tsx src/bin/backfill-reddit.ts", + "backfill:discord": "tsx src/bin/backfill-discord.ts" }, "devDependencies": { "@cloudflare/vitest-pool-workers": "^0.8.0", diff --git a/backend/src/bin/backfill-discord.ts b/backend/src/bin/backfill-discord.ts new file mode 100644 index 0000000..171f860 --- /dev/null +++ b/backend/src/bin/backfill-discord.ts @@ -0,0 +1,192 @@ +import { DateTime } from 'luxon'; +import { Metric } from 'src/interfaces/metrics.interface'; +import { DiscordRepository } from 'src/repositories/discord.repository'; +import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provider.repository'; +import { MetricsPushRepository } from 'src/repositories/metrics-push.repository'; + +class DiscordBackfillService { + private discordRepository = new DiscordRepository(); + + constructor( + private metricsRepository: MetricsPushRepository, + private influxProvider: InfluxMetricsPushProvider, + private env: string, + ) {} + + async backfillDiscord(guildId: string, botToken?: string) { + console.log('Starting Discord backfill...'); + console.log(`Guild ID: ${guildId}`); + console.log(`Bot Token provided: ${!!botToken}`); + + // Delete existing time series + await this.deleteTimeSeries('immich_data_repository_discord_.*', this.env); + + if (!botToken) { + console.log('No bot token provided, can only fetch current stats from public API'); + return; + } + + // Verify bot access and fetch member data + console.log('Verifying bot access to guild...'); + try { + const guildInfo = await this.discordRepository.getGuildInfo(guildId, botToken); + console.log(`Successfully connected to guild: ${guildInfo.name}`); + console.log(`Approximate member count: ${guildInfo.approximate_member_count}`); + + // Fetch all members and push historical metrics + const membersByDate = await this.fetchMembersAndBuildTimeline(guildId, botToken); + await this.pushHistoricalMetrics(membersByDate); + } catch (error) { + console.error('Failed to access guild with bot token:', error); + } + + console.log('Discord backfill complete'); + } + + private async fetchMembersAndBuildTimeline(guildId: string, botToken: string): Promise> { + console.log('Fetching all guild members from Discord...'); + + const membersByDate = new Map(); + let after: string | undefined; + let totalMembers = 0; + + while (true) { + try { + const members = await this.discordRepository.getGuildMembers(guildId, botToken, 1000, after); + + if (members.length === 0) { + break; + } + + // Process each member's join date + for (const member of members) { + if (!member.user || !member.joined_at) { + continue; + } + + const joinDate = DateTime.fromISO(member.joined_at).startOf('day'); + membersByDate.set(joinDate, (membersByDate.get(joinDate) || 0) + 1); + } + + totalMembers += members.length; + console.log(`Fetched ${totalMembers} members so far...`); + + // Get the last member's ID for pagination + const lastMember = members[members.length - 1]; + after = lastMember.user?.id; + + // Rate limiting + await new Promise((resolve) => setTimeout(resolve, 200)); + } catch (error: any) { + // Handle rate limiting + if (error.message?.includes('429')) { + const retryAfter = this.extractRetryAfter(error.message); + console.log(`Rate limited. Waiting ${retryAfter}ms...`); + await new Promise((resolve) => setTimeout(resolve, retryAfter)); + continue; + } + throw error; + } + } + + console.log(`Found ${totalMembers} total members in guild`); + return membersByDate; + } + + private async pushHistoricalMetrics(membersByDate: Map) { + console.log('Pushing historical metrics to InfluxDB...'); + + // Sort dates and build cumulative count + const sortedDates = Array.from(membersByDate.keys()).sort((a, b) => a.toMillis() - b.toMillis()); + + let cumulativeCount = 0; + let metricsCount = 0; + + for (const date of sortedDates) { + cumulativeCount += membersByDate.get(date) || 0; + + const metric = new Metric('discord_member') + .setTimestamp(date.toJSDate()) + .intField('total', cumulativeCount) + .addTag('environment', this.env); + + this.metricsRepository.push(metric); + metricsCount++; + + // Flush every 100 metrics + if (metricsCount % 100 === 0) { + await this.influxProvider.flush(); + console.log(`Flushed ${metricsCount} metrics...`); + } + } + + await this.influxProvider.flush(); + console.log(`Total historical metrics pushed: ${metricsCount}`); + } + + private extractRetryAfter(errorMessage: string): number { + try { + const match = errorMessage.match(/"retry_after":\s*([\d.]+)/); + if (match) { + return Math.ceil(parseFloat(match[1]) * 1000) + 100; + } + } catch { + // Ignore parsing errors + } + return 1000; // Default to 1 second + } + + private async deleteTimeSeries(namePattern: string, environment: string) { + const url = new URL(`${process.env.VMETRICS_DATA_API_URL}/api/v1/admin/tsdb/delete_series`); + const match = `{__name__=~'${namePattern}',environment='${environment}'}`; + url.search = new URLSearchParams({ ['match[]']: match }).toString(); + + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${process.env.VMETRICS_DATA_ADMIN_TOKEN}`, + 'Content-Type': 'application/json', + }, + }); + + console.log(`Deleting time series ${match}`); + if (response.status !== 204) { + console.log(response.status); + throw new Error(`Failed to delete time series ${match}`); + } + } +} + +// Main execution +const main = async () => { + if ( + !process.env.VMETRICS_DATA_WRITE_TOKEN || + !process.env.VMETRICS_DATA_API_URL || + !process.env.VMETRICS_DATA_ADMIN_TOKEN + ) { + throw new Error('Missing VMetrics API credentials'); + } + + if (!process.env.ENVIRONMENT) { + throw new Error('Missing environment'); + } + + const env = `${process.env.ENVIRONMENT}${process.env.TF_VAR_stage ? `_${process.env.TF_VAR_stage}` : ''}`; + const influxProvider = new InfluxMetricsPushProvider( + process.env.VMETRICS_DATA_API_URL, + process.env.VMETRICS_DATA_WRITE_TOKEN, + ); + const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]); + + const service = new DiscordBackfillService(metricsRepository, influxProvider, env); + + // Discord configuration + const guildId = process.env.DISCORD_GUILD_ID || '979116623879368755'; // Immich Discord server ID + const botToken = process.env.DISCORD_BOT_TOKEN; + + await service.backfillDiscord(guildId, botToken); +}; + +main() + .then(() => console.log('Discord backfill complete')) + .catch(console.error); diff --git a/backend/src/bin/backfill.ts b/backend/src/bin/backfill.ts index f030e91..86cbcf3 100644 --- a/backend/src/bin/backfill.ts +++ b/backend/src/bin/backfill.ts @@ -168,7 +168,6 @@ export class BackfillService { let opened = 0; let closed = 0; - const metricPromises = []; let count = 0; for (const { diff --git a/backend/src/index.ts b/backend/src/index.ts index edf4c21..21d5a1f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -7,6 +7,7 @@ import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provi import { MetricsPushRepository } from 'src/repositories/metrics-push.repository'; import { MetricsQueryRepository } from 'src/repositories/metrics-query.repository'; import { ApiWorker } from 'src/workers/api.worker'; +import { DiscordIngestWorker } from 'src/workers/discord-ingest.worker'; import { IngestApiWorker } from 'src/workers/ingest-api.worker'; import { IngestProcessorWorker } from 'src/workers/ingest-processor.worker'; import { RedditIngestWorker } from 'src/workers/reddit-ingest.worker'; @@ -19,7 +20,7 @@ const newApiWorker = (request: FetchRequest, env: WorkerEnv, ctx: ExecutionConte const deferredRepository = new CloudflareDeferredRepository(ctx); const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN); deferredRepository.defer(() => influxProvider.flush()); - const metrics = new MetricsQueryRepository(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_READ_TOKEN, 'prod'); + const metrics = new MetricsQueryRepository(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_READ_TOKEN, asEnvTag(env)); return new ApiWorker(metrics); }; @@ -43,6 +44,7 @@ const handleError = (err: Error) => { const router = AutoRouter() .get('/api/github', (...args) => newApiWorker(...args).getGithubData()) .get('/api/reddit', (...args) => newApiWorker(...args).getRedditData()) + .get('/api/discord', (...args) => newApiWorker(...args).getDiscordData()) .post('/ingest/github/:slug', slugAuth, async (req, env) => newIngestApiWorker(req, env) .onGithubEvent(await req.json()) @@ -72,12 +74,16 @@ export default { const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN); const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]); const redditWorker = new RedditIngestWorker(metricsRepository, asEnvTag(env)); + const discordWorker = new DiscordIngestWorker(metricsRepository, asEnvTag(env)); try { - await redditWorker.fetchAndStoreCurrentMetrics('immich'); + await Promise.all([ + redditWorker.fetchAndStoreCurrentMetrics('immich'), + discordWorker.fetchAndStoreCurrentMetrics('immich'), + ]); await influxProvider.flush(); } catch (error) { - console.error('Failed to fetch Reddit metrics:', error); + console.error('Failed to fetch metrics:', error); } }, } satisfies ExportedHandler; diff --git a/backend/src/repositories/discord.repository.ts b/backend/src/repositories/discord.repository.ts new file mode 100644 index 0000000..0f6a335 --- /dev/null +++ b/backend/src/repositories/discord.repository.ts @@ -0,0 +1,118 @@ +// Discord API doesn't work with fetch-retry wrapper in TSX environment +// Use native fetch for all Discord API calls + +export interface DiscordGuildData { + id: string; + name: string; + icon?: string; + banner?: string; + premium_tier: number; + premium_subscription_count: number; + approximate_member_count: number; + approximate_presence_count: number; +} + +export interface DiscordInviteResponse { + code: string; + guild: DiscordGuildData; + approximate_member_count: number; + approximate_presence_count: number; +} + +export interface DiscordUser { + id: string; + username: string; + discriminator: string; + avatar?: string | null; + bot?: boolean; + system?: boolean; + mfa_enabled?: boolean; + banner?: string | null; + accent_color?: number | null; + locale?: string; + verified?: boolean; + email?: string | null; + flags?: number; + premium_type?: number; + public_flags?: number; +} + +export interface DiscordMember { + user?: DiscordUser; + nick?: string | null; + avatar?: string | null; + roles: string[]; + joined_at: string; + premium_since?: string | null; + deaf: boolean; + mute: boolean; + flags: number; + pending?: boolean; + permissions?: string; + communication_disabled_until?: string | null; +} + +export class DiscordRepository { + private readonly apiBase = 'https://discord.com/api/v9'; + private readonly userAgent = 'Mozilla/5.0 (compatible; ImmichDataBot/1.0)'; + + async getGuildInfo(guildId: string, botToken: string): Promise { + const url = `${this.apiBase}/guilds/${guildId}?with_counts=true`; + + // Use native fetch for Discord API to avoid fetch-retry issues + const response = await fetch(url, { + headers: { + Authorization: `Bot ${botToken}`, + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error(`Discord API Error: ${response.status} - ${errorText}`); + throw new Error(`Failed to fetch guild info: ${response.status} ${response.statusText} - ${errorText}`); + } + + return response.json(); + } + + async getGuildStats(inviteCode: string = 'immich'): Promise { + const url = `${this.apiBase}/invites/${inviteCode}?with_counts=true&with_expiration=true`; + + const response = await fetch(url, { + headers: { + 'User-Agent': this.userAgent, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch Discord data: ${response.status} ${response.statusText}`); + } + + return response.json() as Promise; + } + + async getGuildMembers(guildId: string, botToken: string, limit = 1000, after?: string): Promise { + const params = new URLSearchParams({ + limit: limit.toString(), + ...(after && { after }), + }); + + const url = `${this.apiBase}/guilds/${guildId}/members?${params}`; + + const response = await fetch(url, { + headers: { + Authorization: `Bot ${botToken}`, + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error(`Discord API Error: ${response.status} - ${errorText}`); + throw new Error(`Failed to fetch guild members: ${response.status} ${response.statusText} - ${errorText}`); + } + + return response.json() as Promise; + } +} diff --git a/backend/src/workers/api.worker.ts b/backend/src/workers/api.worker.ts index 78a1238..b5009c0 100644 --- a/backend/src/workers/api.worker.ts +++ b/backend/src/workers/api.worker.ts @@ -22,6 +22,12 @@ export class ApiWorker { }); } + async getDiscordData() { + return this.asTimeSeries({ + members: { metricName: 'immich_data_repository_discord_member_total', labels: [] }, + }); + } + async asTimeSeries(metrics: Record) { const promises = Object.entries(metrics).map(async ([key, value]) => { const results = await this.metricsRepository.queryMaxOverTime({ diff --git a/backend/src/workers/discord-ingest.worker.ts b/backend/src/workers/discord-ingest.worker.ts new file mode 100644 index 0000000..f4815ca --- /dev/null +++ b/backend/src/workers/discord-ingest.worker.ts @@ -0,0 +1,44 @@ +import { IMetricsPushRepository, Metric } from 'src/interfaces/metrics.interface'; +import { DiscordRepository } from 'src/repositories/discord.repository'; + +export class DiscordIngestWorker { + private discordRepository = new DiscordRepository(); + + constructor( + private metricsRepository: IMetricsPushRepository, + private envTag: string, + ) {} + + async fetchAndStoreCurrentMetrics(inviteCode: string = 'immich') { + try { + const data = await this.discordRepository.getGuildStats(inviteCode); + + const metrics = [ + new Metric('discord_member') + .intField('total', data.approximate_member_count) + .addTag('environment', this.envTag), + new Metric('discord_online') + .intField('total', data.approximate_presence_count) + .addTag('environment', this.envTag), + new Metric('discord_nitro') + .intField('total', data.guild.premium_subscription_count) + .addTag('environment', this.envTag), + new Metric('discord_server_tier') + .intField('level', data.guild.premium_tier) + .addTag('environment', this.envTag), + ]; + + metrics.forEach((metric) => this.metricsRepository.push(metric)); + + return { + members: data.approximate_member_count, + online: data.approximate_presence_count, + nitro: data.guild.premium_subscription_count, + tier: data.guild.premium_tier, + }; + } catch (error) { + console.error(`Failed to fetch Discord data for invite ${inviteCode}:`, error); + throw error; + } + } +} \ No newline at end of file diff --git a/frontend/src/lib/services/api.svelte.ts b/frontend/src/lib/services/api.svelte.ts index caa596d..632c483 100644 --- a/frontend/src/lib/services/api.svelte.ts +++ b/frontend/src/lib/services/api.svelte.ts @@ -32,7 +32,18 @@ type RedditDataResponse = { subscribers: ReportData; }; +type DiscordDataResponse = { + members: ReportData; + online: ReportData; + nitro: ReportData; + serverTier: ReportData; +}; + export const redditMembers = $state<{ value?: DataRecord[] }>({}); +export const discordMembers = $state<{ value?: DataRecord[] }>({}); +export const discordOnline = $state<{ value?: DataRecord[] }>({}); +export const discordNitro = $state<{ value?: DataRecord[] }>({}); +export const discordServerTier = $state<{ value?: DataRecord[] }>({}); export const loadRedditData = async () => { const response = await fetch('/api/reddit'); @@ -41,3 +52,14 @@ export const loadRedditData = async () => { redditMembers.value = subscribers; } }; + +export const loadDiscordData = async () => { + const response = await fetch('/api/discord'); + if (response.ok) { + const { members, online, nitro, serverTier } = (await response.json()) as DiscordDataResponse; + discordMembers.value = members; + discordOnline.value = online; + discordNitro.value = nitro; + discordServerTier.value = serverTier; + } +}; diff --git a/frontend/src/routes/+page.svelte b/frontend/src/routes/+page.svelte index 1dd07b8..6f075ba 100644 --- a/frontend/src/routes/+page.svelte +++ b/frontend/src/routes/+page.svelte @@ -10,6 +10,8 @@ loadGithubData, loadRedditData, redditMembers, + loadDiscordData, + discordMembers, } from '$lib/services/api.svelte'; import { Card, @@ -45,6 +47,7 @@ void loadGithubData(); void loadRedditData(); + void loadDiscordData(); @@ -196,8 +199,8 @@
- Reddit - r/immich + Community + r/immich • discord.gg/immich
@@ -205,7 +208,7 @@ - Total Members + Reddit Members @@ -219,6 +222,25 @@ /> + + + + + + Discord Members + + + + + + +