feat: discord stats (#188)

backend/package.json

@@ -16,7 +16,8 @@
     "format": "prettier --check .",
     "format:fix": "prettier --write .",
     "check": "tsc --noEmit",
-    "backfill:reddit": "tsx src/bin/backfill-reddit.ts"
+    "backfill:reddit": "tsx src/bin/backfill-reddit.ts",
+    "backfill:discord": "tsx src/bin/backfill-discord.ts"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.8.0",

backend/src/bin/backfill-discord.ts

@@ -0,0 +1,192 @@
+import { DateTime } from 'luxon';
+import { Metric } from 'src/interfaces/metrics.interface';
+import { DiscordRepository } from 'src/repositories/discord.repository';
+import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provider.repository';
+import { MetricsPushRepository } from 'src/repositories/metrics-push.repository';
+
+class DiscordBackfillService {
+  private discordRepository = new DiscordRepository();
+
+  constructor(
+    private metricsRepository: MetricsPushRepository,
+    private influxProvider: InfluxMetricsPushProvider,
+    private env: string,
+  ) {}
+
+  async backfillDiscord(guildId: string, botToken?: string) {
+    console.log('Starting Discord backfill...');
+    console.log(`Guild ID: ${guildId}`);
+    console.log(`Bot Token provided: ${!!botToken}`);
+
+    // Delete existing time series
+    await this.deleteTimeSeries('immich_data_repository_discord_.*', this.env);
+
+    if (!botToken) {
+      console.log('No bot token provided, can only fetch current stats from public API');
+      return;
+    }
+
+    // Verify bot access and fetch member data
+    console.log('Verifying bot access to guild...');
+    try {
+      const guildInfo = await this.discordRepository.getGuildInfo(guildId, botToken);
+      console.log(`Successfully connected to guild: ${guildInfo.name}`);
+      console.log(`Approximate member count: ${guildInfo.approximate_member_count}`);
+
+      // Fetch all members and push historical metrics
+      const membersByDate = await this.fetchMembersAndBuildTimeline(guildId, botToken);
+      await this.pushHistoricalMetrics(membersByDate);
+    } catch (error) {
+      console.error('Failed to access guild with bot token:', error);
+    }
+
+    console.log('Discord backfill complete');
+  }
+
+  private async fetchMembersAndBuildTimeline(guildId: string, botToken: string): Promise<Map<DateTime, number>> {
+    console.log('Fetching all guild members from Discord...');
+
+    const membersByDate = new Map<DateTime, number>();
+    let after: string | undefined;
+    let totalMembers = 0;
+
+    while (true) {
+      try {
+        const members = await this.discordRepository.getGuildMembers(guildId, botToken, 1000, after);
+
+        if (members.length === 0) {
+          break;
+        }
+
+        // Process each member's join date
+        for (const member of members) {
+          if (!member.user || !member.joined_at) {
+            continue;
+          }
+
+          const joinDate = DateTime.fromISO(member.joined_at).startOf('day');
+          membersByDate.set(joinDate, (membersByDate.get(joinDate) || 0) + 1);
+        }
+
+        totalMembers += members.length;
+        console.log(`Fetched ${totalMembers} members so far...`);
+
+        // Get the last member's ID for pagination
+        const lastMember = members[members.length - 1];
+        after = lastMember.user?.id;
+
+        // Rate limiting
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      } catch (error: any) {
+        // Handle rate limiting
+        if (error.message?.includes('429')) {
+          const retryAfter = this.extractRetryAfter(error.message);
+          console.log(`Rate limited. Waiting ${retryAfter}ms...`);
+          await new Promise((resolve) => setTimeout(resolve, retryAfter));
+          continue;
+        }
+        throw error;
+      }
+    }
+
+    console.log(`Found ${totalMembers} total members in guild`);
+    return membersByDate;
+  }
+
+  private async pushHistoricalMetrics(membersByDate: Map<DateTime, number>) {
+    console.log('Pushing historical metrics to InfluxDB...');
+
+    // Sort dates and build cumulative count
+    const sortedDates = Array.from(membersByDate.keys()).sort((a, b) => a.toMillis() - b.toMillis());
+
+    let cumulativeCount = 0;
+    let metricsCount = 0;
+
+    for (const date of sortedDates) {
+      cumulativeCount += membersByDate.get(date) || 0;
+
+      const metric = new Metric('discord_member')
+        .setTimestamp(date.toJSDate())
+        .intField('total', cumulativeCount)
+        .addTag('environment', this.env);
+
+      this.metricsRepository.push(metric);
+      metricsCount++;
+
+      // Flush every 100 metrics
+      if (metricsCount % 100 === 0) {
+        await this.influxProvider.flush();
+        console.log(`Flushed ${metricsCount} metrics...`);
+      }
+    }
+
+    await this.influxProvider.flush();
+    console.log(`Total historical metrics pushed: ${metricsCount}`);
+  }
+
+  private extractRetryAfter(errorMessage: string): number {
+    try {
+      const match = errorMessage.match(/"retry_after":\s*([\d.]+)/);
+      if (match) {
+        return Math.ceil(parseFloat(match[1]) * 1000) + 100;
+      }
+    } catch {
+      // Ignore parsing errors
+    }
+    return 1000; // Default to 1 second
+  }
+
+  private async deleteTimeSeries(namePattern: string, environment: string) {
+    const url = new URL(`${process.env.VMETRICS_DATA_API_URL}/api/v1/admin/tsdb/delete_series`);
+    const match = `{__name__=~'${namePattern}',environment='${environment}'}`;
+    url.search = new URLSearchParams({ ['match[]']: match }).toString();
+
+    const response = await fetch(url, {
+      method: 'GET',
+      headers: {
+        Authorization: `Bearer ${process.env.VMETRICS_DATA_ADMIN_TOKEN}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    console.log(`Deleting time series ${match}`);
+    if (response.status !== 204) {
+      console.log(response.status);
+      throw new Error(`Failed to delete time series ${match}`);
+    }
+  }
+}
+
+// Main execution
+const main = async () => {
+  if (
+    !process.env.VMETRICS_DATA_WRITE_TOKEN ||
+    !process.env.VMETRICS_DATA_API_URL ||
+    !process.env.VMETRICS_DATA_ADMIN_TOKEN
+  ) {
+    throw new Error('Missing VMetrics API credentials');
+  }
+
+  if (!process.env.ENVIRONMENT) {
+    throw new Error('Missing environment');
+  }
+
+  const env = `${process.env.ENVIRONMENT}${process.env.TF_VAR_stage ? `_${process.env.TF_VAR_stage}` : ''}`;
+  const influxProvider = new InfluxMetricsPushProvider(
+    process.env.VMETRICS_DATA_API_URL,
+    process.env.VMETRICS_DATA_WRITE_TOKEN,
+  );
+  const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]);
+
+  const service = new DiscordBackfillService(metricsRepository, influxProvider, env);
+
+  // Discord configuration
+  const guildId = process.env.DISCORD_GUILD_ID || '979116623879368755'; // Immich Discord server ID
+  const botToken = process.env.DISCORD_BOT_TOKEN;
+
+  await service.backfillDiscord(guildId, botToken);
+};
+
+main()
+  .then(() => console.log('Discord backfill complete'))
+  .catch(console.error);

backend/src/bin/backfill.ts

@@ -168,7 +168,6 @@ export class BackfillService {
     let opened = 0;
     let closed = 0;
 
-    const metricPromises = [];
     let count = 0;
 
     for (const {

backend/src/index.ts

@@ -7,6 +7,7 @@ import { InfluxMetricsPushProvider } from 'src/repositories/influx-metrics-provi
 import { MetricsPushRepository } from 'src/repositories/metrics-push.repository';
 import { MetricsQueryRepository } from 'src/repositories/metrics-query.repository';
 import { ApiWorker } from 'src/workers/api.worker';
+import { DiscordIngestWorker } from 'src/workers/discord-ingest.worker';
 import { IngestApiWorker } from 'src/workers/ingest-api.worker';
 import { IngestProcessorWorker } from 'src/workers/ingest-processor.worker';
 import { RedditIngestWorker } from 'src/workers/reddit-ingest.worker';
@@ -19,7 +20,7 @@ const newApiWorker = (request: FetchRequest, env: WorkerEnv, ctx: ExecutionConte
   const deferredRepository = new CloudflareDeferredRepository(ctx);
   const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN);
   deferredRepository.defer(() => influxProvider.flush());
-  const metrics = new MetricsQueryRepository(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_READ_TOKEN, 'prod');
+  const metrics = new MetricsQueryRepository(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_READ_TOKEN, asEnvTag(env));
 
   return new ApiWorker(metrics);
 };
@@ -43,6 +44,7 @@ const handleError = (err: Error) => {
 const router = AutoRouter<FetchRequest, [WorkerEnv, ExecutionContext]>()
   .get('/api/github', (...args) => newApiWorker(...args).getGithubData())
   .get('/api/reddit', (...args) => newApiWorker(...args).getRedditData())
+  .get('/api/discord', (...args) => newApiWorker(...args).getDiscordData())
   .post('/ingest/github/:slug', slugAuth, async (req, env) =>
     newIngestApiWorker(req, env)
       .onGithubEvent(await req.json())
@@ -72,12 +74,16 @@ export default {
     const influxProvider = new InfluxMetricsPushProvider(env.VMETRICS_DATA_API_URL, env.VMETRICS_DATA_WRITE_TOKEN);
     const metricsRepository = new MetricsPushRepository('immich_data_repository', {}, [influxProvider]);
     const redditWorker = new RedditIngestWorker(metricsRepository, asEnvTag(env));
+    const discordWorker = new DiscordIngestWorker(metricsRepository, asEnvTag(env));
 
     try {
-      await redditWorker.fetchAndStoreCurrentMetrics('immich');
+      await Promise.all([
+        redditWorker.fetchAndStoreCurrentMetrics('immich'),
+        discordWorker.fetchAndStoreCurrentMetrics('immich'),
+      ]);
       await influxProvider.flush();
     } catch (error) {
-      console.error('Failed to fetch Reddit metrics:', error);
+      console.error('Failed to fetch metrics:', error);
     }
   },
 } satisfies ExportedHandler<WorkerEnv, QueueItem>;

backend/src/repositories/discord.repository.ts

@@ -0,0 +1,118 @@
+// Discord API doesn't work with fetch-retry wrapper in TSX environment
+// Use native fetch for all Discord API calls
+
+export interface DiscordGuildData {
+  id: string;
+  name: string;
+  icon?: string;
+  banner?: string;
+  premium_tier: number;
+  premium_subscription_count: number;
+  approximate_member_count: number;
+  approximate_presence_count: number;
+}
+
+export interface DiscordInviteResponse {
+  code: string;
+  guild: DiscordGuildData;
+  approximate_member_count: number;
+  approximate_presence_count: number;
+}
+
+export interface DiscordUser {
+  id: string;
+  username: string;
+  discriminator: string;
+  avatar?: string | null;
+  bot?: boolean;
+  system?: boolean;
+  mfa_enabled?: boolean;
+  banner?: string | null;
+  accent_color?: number | null;
+  locale?: string;
+  verified?: boolean;
+  email?: string | null;
+  flags?: number;
+  premium_type?: number;
+  public_flags?: number;
+}
+
+export interface DiscordMember {
+  user?: DiscordUser;
+  nick?: string | null;
+  avatar?: string | null;
+  roles: string[];
+  joined_at: string;
+  premium_since?: string | null;
+  deaf: boolean;
+  mute: boolean;
+  flags: number;
+  pending?: boolean;
+  permissions?: string;
+  communication_disabled_until?: string | null;
+}
+
+export class DiscordRepository {
+  private readonly apiBase = 'https://discord.com/api/v9';
+  private readonly userAgent = 'Mozilla/5.0 (compatible; ImmichDataBot/1.0)';
+
+  async getGuildInfo(guildId: string, botToken: string): Promise<any> {
+    const url = `${this.apiBase}/guilds/${guildId}?with_counts=true`;
+
+    // Use native fetch for Discord API to avoid fetch-retry issues
+    const response = await fetch(url, {
+      headers: {
+        Authorization: `Bot ${botToken}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      console.error(`Discord API Error: ${response.status} - ${errorText}`);
+      throw new Error(`Failed to fetch guild info: ${response.status} ${response.statusText} - ${errorText}`);
+    }
+
+    return response.json();
+  }
+
+  async getGuildStats(inviteCode: string = 'immich'): Promise<DiscordInviteResponse> {
+    const url = `${this.apiBase}/invites/${inviteCode}?with_counts=true&with_expiration=true`;
+
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': this.userAgent,
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch Discord data: ${response.status} ${response.statusText}`);
+    }
+
+    return response.json() as Promise<DiscordInviteResponse>;
+  }
+
+  async getGuildMembers(guildId: string, botToken: string, limit = 1000, after?: string): Promise<DiscordMember[]> {
+    const params = new URLSearchParams({
+      limit: limit.toString(),
+      ...(after && { after }),
+    });
+
+    const url = `${this.apiBase}/guilds/${guildId}/members?${params}`;
+
+    const response = await fetch(url, {
+      headers: {
+        Authorization: `Bot ${botToken}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      console.error(`Discord API Error: ${response.status} - ${errorText}`);
+      throw new Error(`Failed to fetch guild members: ${response.status} ${response.statusText} - ${errorText}`);
+    }
+
+    return response.json() as Promise<DiscordMember[]>;
+  }
+}

backend/src/workers/api.worker.ts

@@ -22,6 +22,12 @@ export class ApiWorker {
     });
   }
 
+  async getDiscordData() {
+    return this.asTimeSeries({
+      members: { metricName: 'immich_data_repository_discord_member_total', labels: [] },
+    });
+  }
+
   async asTimeSeries(metrics: Record<string, { metricName: string; labels: string[] }>) {
     const promises = Object.entries(metrics).map(async ([key, value]) => {
       const results = await this.metricsRepository.queryMaxOverTime({

backend/src/workers/discord-ingest.worker.ts

@@ -0,0 +1,44 @@
+import { IMetricsPushRepository, Metric } from 'src/interfaces/metrics.interface';
+import { DiscordRepository } from 'src/repositories/discord.repository';
+
+export class DiscordIngestWorker {
+  private discordRepository = new DiscordRepository();
+
+  constructor(
+    private metricsRepository: IMetricsPushRepository,
+    private envTag: string,
+  ) {}
+
+  async fetchAndStoreCurrentMetrics(inviteCode: string = 'immich') {
+    try {
+      const data = await this.discordRepository.getGuildStats(inviteCode);
+
+      const metrics = [
+        new Metric('discord_member')
+          .intField('total', data.approximate_member_count)
+          .addTag('environment', this.envTag),
+        new Metric('discord_online')
+          .intField('total', data.approximate_presence_count)
+          .addTag('environment', this.envTag),
+        new Metric('discord_nitro')
+          .intField('total', data.guild.premium_subscription_count)
+          .addTag('environment', this.envTag),
+        new Metric('discord_server_tier')
+          .intField('level', data.guild.premium_tier)
+          .addTag('environment', this.envTag),
+      ];
+
+      metrics.forEach((metric) => this.metricsRepository.push(metric));
+
+      return {
+        members: data.approximate_member_count,
+        online: data.approximate_presence_count,
+        nitro: data.guild.premium_subscription_count,
+        tier: data.guild.premium_tier,
+      };
+    } catch (error) {
+      console.error(`Failed to fetch Discord data for invite ${inviteCode}:`, error);
+      throw error;
+    }
+  }
+}

frontend/src/lib/services/api.svelte.ts

@@ -32,7 +32,18 @@ type RedditDataResponse = {
   subscribers: ReportData;
 };
 
+type DiscordDataResponse = {
+  members: ReportData;
+  online: ReportData;
+  nitro: ReportData;
+  serverTier: ReportData;
+};
+
 export const redditMembers = $state<{ value?: DataRecord[] }>({});
+export const discordMembers = $state<{ value?: DataRecord[] }>({});
+export const discordOnline = $state<{ value?: DataRecord[] }>({});
+export const discordNitro = $state<{ value?: DataRecord[] }>({});
+export const discordServerTier = $state<{ value?: DataRecord[] }>({});
 
 export const loadRedditData = async () => {
   const response = await fetch('/api/reddit');
@@ -41,3 +52,14 @@ export const loadRedditData = async () => {
     redditMembers.value = subscribers;
   }
 };
+
+export const loadDiscordData = async () => {
+  const response = await fetch('/api/discord');
+  if (response.ok) {
+    const { members, online, nitro, serverTier } = (await response.json()) as DiscordDataResponse;
+    discordMembers.value = members;
+    discordOnline.value = online;
+    discordNitro.value = nitro;
+    discordServerTier.value = serverTier;
+  }
+};

frontend/src/routes/+page.svelte

@@ -10,6 +10,8 @@
     loadGithubData,
     loadRedditData,
     redditMembers,
+    loadDiscordData,
+    discordMembers,
   } from '$lib/services/api.svelte';
   import {
     Card,
@@ -45,6 +47,7 @@
 
   void loadGithubData();
   void loadRedditData();
+  void loadDiscordData();
 </script>
 
 <Container size="giant" center>
@@ -196,8 +199,8 @@
 
     <section class="flex flex-col gap-4">
       <div>
-        <Heading size="large" tag="h1">Reddit</Heading>
-        <Text color="muted">r/immich</Text>
+        <Heading size="large" tag="h1">Community</Heading>
+        <Text color="muted">r/immich • discord.gg/immich</Text>
       </div>
       <div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
         <Card color="secondary">
@@ -205,7 +208,7 @@
             <CardTitle>
               <HStack>
                 <Icon icon={mdiAccountGroup} />
-                <span>Total Members</span>
+                <span>Reddit Members</span>
               </HStack>
             </CardTitle>
           </CardHeader>
@@ -219,6 +222,25 @@
             />
           </CardBody>
         </Card>
+        <Card color="secondary">
+          <CardHeader>
+            <CardTitle>
+              <HStack>
+                <Icon icon={mdiAccountGroup} />
+                <span>Discord Members</span>
+              </HStack>
+            </CardTitle>
+          </CardHeader>
+          <CardBody>
+            <GithubGraph
+              color="purple"
+              id="discord-members"
+              data={discordMembers.value}
+              {cursorOpts}
+              label="Discord members"
+            />
+          </CardBody>
+        </Card>
       </div>
     </section>
   </Stack>