feat: use .well-known/yucca.json to discover backend (#126)

packages/yucca-sdk/orchestration-api/src/backends/backend.ts

@@ -7,7 +7,6 @@ import {
   RepositoryUpdateResponseDto,
 } from '@futo-org/backups-api-client';
 import { BackendType } from '../enum';
-import { ModuleConfig } from '../moduleConfig';
 import { BackendConfiguration } from '../schema/tables/backend.table';
 
 export abstract class Backend {
@@ -30,13 +29,10 @@ export abstract class Backend {
   abstract submitMetricRepositorySize(id: string, size: number): Promise<void>;
   abstract submitStructuredLog(summary: string, data: object): void;
 
-  static from(configuration: BackendConfiguration, moduleConfig: ModuleConfig) {
+  static from(configuration: BackendConfiguration) {
     switch (configuration.type) {
       case BackendType.Yucca: {
-        return new YuccaBackend({
+        return new YuccaBackend(configuration);
-          url: moduleConfig.yuccaProductionApi,
-          ...configuration,
-        });
       }
       case BackendType.Local: {
         return new LocalBackend(configuration);

packages/yucca-sdk/orchestration-api/src/backends/yucca.backend.ts

@@ -13,18 +13,49 @@ import {
   submitStructuredLog,
   updateRepository,
 } from '@futo-org/backups-api-client';
+import { YUCCA_WELL_KNOWN } from '../const';
 import { BackendType, CookieName } from '../enum';
 import { BackendConfiguration } from '../schema/tables/backend.table';
 import { Backend } from './backend';
 
+type WellKnown = {
+  backends: Record<string, { displayName: string; api: string }>;
+  defaultBackend: string;
+};
+
+class YuccaWellKnown {
+  private data?: WellKnown;
+
+  async get() {
+    this.data ??= await fetch(YUCCA_WELL_KNOWN).then((response) => response.json());
+    return this.data! as WellKnown;
+  }
+
+  async getBaseUrlById(id: string) {
+    const { backends } = await this.get();
+    return backends[id].api;
+  }
+
+  async getBaseUrl() {
+    const { backends, defaultBackend } = await this.get();
+    return backends[defaultBackend].api;
+  }
+}
+
+export const yuccaWellKnown = new YuccaWellKnown();
+
 export class YuccaBackend extends Backend {
-  constructor(protected readonly configuration: BackendConfiguration & { type: BackendType.Yucca; url: string }) {
+  constructor(protected readonly configuration: BackendConfiguration & { type: BackendType.Yucca; url?: string }) {
     super(configuration);
   }
 
-  private get requestOptions() {
+  private async getRequestOptions() {
     return {
-      baseUrl: this.configuration.url,
+      baseUrl:
+        this.configuration.url ??
+        (this.configuration.uuid
+          ? await yuccaWellKnown.getBaseUrlById(this.configuration.uuid)
+          : await yuccaWellKnown.getBaseUrl()),
       headers: {
         cookie: `${CookieName.YuccaAccessToken}=${this.configuration.accessToken}`,
       },
@@ -40,60 +71,56 @@ export class YuccaBackend extends Backend {
   }
 
   async checkOnline() {
-    await getAuth(this.requestOptions);
+    await getAuth(await this.getRequestOptions());
   }
 
-  createRepository(dto: RepositoryCreateRequestDto) {
+  async createRepository(dto: RepositoryCreateRequestDto) {
-    return createRepository(dto, this.requestOptions);
+    return await createRepository(dto, await this.getRequestOptions());
   }
 
-  updateRepository(id: string, dto: RepositoryUpdateRequestDto) {
+  async updateRepository(id: string, dto: RepositoryUpdateRequestDto) {
-    return updateRepository(id, dto, this.requestOptions);
+    return await updateRepository(id, dto, await this.getRequestOptions());
   }
 
-  getRepository(id: string) {
+  async getRepository(id: string) {
-    return getRepository(id, this.requestOptions);
+    return await getRepository(id, await this.getRequestOptions());
   }
 
-  getRepositories() {
+  async getRepositories() {
-    return getRepositories(this.requestOptions);
+    return await getRepositories(await this.getRequestOptions());
   }
 
-  deleteRepository(id: string) {
+  async deleteRepository(id: string) {
-    return deleteRepository(id, this.requestOptions);
+    return await deleteRepository(id, await this.getRequestOptions());
   }
 
   async getResticEndpoint(id: string) {
-    const { url } = await createResticUrl(id, this.requestOptions);
+    const { url } = await createResticUrl(id, await this.getRequestOptions());
     return url;
   }
 
-  submitMetricBackupStart(id: string): Promise<void> {
+  async submitMetricBackupStart(id: string): Promise<void> {
-    return submitMetricBackupStart(id, this.requestOptions);
+    return await submitMetricBackupStart(id, await this.getRequestOptions());
   }
 
-  submitMetricBackupEnd(id: string, success: boolean, durationMs: number): Promise<void> {
+  async submitMetricBackupEnd(id: string, success: boolean, durationMs: number): Promise<void> {
-    return submitMetricBackupEnd(
+    return await submitMetricBackupEnd(
       id,
       {
         durationMs,
         success,
       },
-      this.requestOptions,
+      await this.getRequestOptions(),
     );
   }
 
-  submitMetricRepositorySize(id: string, sizeBytes: number): Promise<void> {
+  async submitMetricRepositorySize(id: string, sizeBytes: number): Promise<void> {
-    return submitMetricRepositorySize(id, { sizeBytes }, this.requestOptions);
+    return await submitMetricRepositorySize(id, { sizeBytes }, await this.getRequestOptions());
   }
 
   submitStructuredLog(summary: string, data: object) {
-    void submitStructuredLog(
+    void this.getRequestOptions()
-      {
+      .then((requestOptions) => submitStructuredLog({ summary, data }, requestOptions))
-        summary,
+      .catch((error) => console.error('Failed to submit log', error));
-        data,
-      },
-      this.requestOptions,
-    ).catch((error) => console.error('Failed to submit log', error));
   }
 }

packages/yucca-sdk/orchestration-api/src/const.ts

@@ -1,2 +1,3 @@
 export const ORCHESTRATION_PORT = 22_676;
-export const YUCCA_PRODUCTION_UUID = 'd0368cdd-39ae-40e1-91d6-d81815c65c7e';
+export const REPOSITORY_DEFAULT_CLOUD_UUID = 'd0368cdd-39ae-40e1-91d6-d81815c65c7e';
+export const YUCCA_WELL_KNOWN = 'https://futo.cloud/.well-known/yucca.json';

packages/yucca-sdk/orchestration-api/src/moduleConfig.ts

@@ -13,7 +13,7 @@ export type ImmichIntegration = {
 
 export type ModuleConfig = {
   statePath: string;
-  yuccaProductionApi: string;
+  yuccaProductionApi?: string;
   externalBaseUrl?: string;
   requireWsAuth?: boolean;
   requireLock?: boolean;

packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts

@@ -95,13 +95,9 @@ export const services = [
 
 @Module({})
 export class OrchestrationApiModule {
-  static forRoot(config: Partial<ModuleConfig> & Pick<ModuleConfig, 'yuccaProductionApi'>): DynamicModule {
+  static forRoot(config: Partial<ModuleConfig>): DynamicModule {
     config.statePath ??= resolve(homedir(), '.yucca');
 
-    if (!config.yuccaProductionApi) {
-      throw new Error('config.yuccaProductionApi is missing');
-    }
-
     return {
       module: OrchestrationApiModule,
       imports: [

packages/yucca-sdk/orchestration-api/src/repositories/backend.repository.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { Kysely } from 'kysely';
 import { InjectKysely } from 'nestjs-kysely';
+import { Backend } from '../backends/backend';
 import { DB } from '../schema';
 import { BackendConfiguration } from '../schema/tables/backend.table';
 
@@ -22,10 +23,12 @@ export class BackendRepository {
 
   async getBackends() {
     const backends = await this.db.selectFrom('backends').selectAll('backends').execute();
-    return backends.map(({ id, configuration }) => ({
+    return backends
-      id,
+      .map(({ id, configuration }) => ({
-      configuration: JSON.parse(configuration) as BackendConfiguration,
+        id,
-    }));
+        configuration: JSON.parse(configuration) as BackendConfiguration,
+      }))
+      .map(({ id, configuration }) => ({ id, configuration, backend: Backend.from(configuration) }));
   }
 
   async getBackend(id: string) {
@@ -35,9 +38,12 @@ export class BackendRepository {
       .where('id', '=', id)
       .executeTakeFirstOrThrow();
 
+    const configuration = JSON.parse(backend.configuration) as BackendConfiguration;
+
     return {
       id: backend.id,
-      configuration: JSON.parse(backend.configuration) as BackendConfiguration,
+      configuration,
+      backend: Backend.from(configuration),
     };
   }
 }

packages/yucca-sdk/orchestration-api/src/schema/tables/backend.table.ts

@@ -11,6 +11,7 @@ export type BackendConfiguration =
        */
       type: BackendType.Yucca;
       url?: string;
+      uuid?: string;
       accessToken: string;
     }
   | {

packages/yucca-sdk/orchestration-api/src/services/auth.service.ts

@@ -1,6 +1,7 @@
 import { Injectable, InternalServerErrorException } from '@nestjs/common';
 import { createEventSource, EventSourceClient } from 'eventsource-client';
-import { YUCCA_PRODUCTION_UUID } from '../const';
+import { yuccaWellKnown } from '../backends/yucca.backend';
+import { REPOSITORY_DEFAULT_CLOUD_UUID } from '../const';
 import { BackendType } from '../enum';
 import { EventsGateway } from '../events/events.gateway';
 import { BackendRepository } from '../repositories/backend.repository';
@@ -18,25 +19,26 @@ export class AuthService {
     readonly telemetry: TelemetryService,
   ) {}
 
-  private async waitForDeviceFlow(events: EventSourceClient) {
+  private async waitForDeviceFlow(events: EventSourceClient, url?: string) {
     for await (const { data } of events) {
       const { type, accessToken } = JSON.parse(data);
 
       switch (type) {
         case 'SUCCESS': {
-          await this.backend.updateBackend(YUCCA_PRODUCTION_UUID, {
+          await this.backend.updateBackend(REPOSITORY_DEFAULT_CLOUD_UUID, {
             type: BackendType.Yucca,
             accessToken,
+            url,
           });
 
           this.telemetry.submitStructuredLog('Connected FUTO Backups backend', {
-            backendId: YUCCA_PRODUCTION_UUID,
+            backendId: REPOSITORY_DEFAULT_CLOUD_UUID,
           });
 
           this.events.publish({
             type: 'BackendCreate',
             backend: {
-              id: YUCCA_PRODUCTION_UUID,
+              id: REPOSITORY_DEFAULT_CLOUD_UUID,
               type: BackendType.Yucca,
               description: 'FUTO Backups',
               isOnline: true,
@@ -61,15 +63,18 @@ export class AuthService {
   }
 
   async oidcDeviceFlow(): Promise<{ userCode: string; verificationUri: string }> {
+    const overrideEndpoint = this.moduleConfig.get().yuccaProductionApi;
+    const endpoint = overrideEndpoint ?? (await yuccaWellKnown.getBaseUrl());
+
     const events: EventSourceClient = createEventSource({
-      url: new URL('/api/auth/oidc/device', this.moduleConfig.get().yuccaProductionApi),
+      url: new URL('/api/auth/oidc/device', endpoint),
       onDisconnect: () => events.close(),
     });
 
     for await (const { data } of events) {
       const { userCode, verificationUri } = JSON.parse(data);
 
-      void this.waitForDeviceFlow(events).catch(() => {});
+      void this.waitForDeviceFlow(events, overrideEndpoint).catch(() => {});
 
       return {
         userCode,

packages/yucca-sdk/orchestration-api/src/services/backend.service.ts

@@ -5,13 +5,11 @@ import { BackendResponseDto, BackendsResponseDto, CreateLocalBackendRequestDto }
 import { BackendType } from '../enum';
 import { EventsGateway } from '../events/events.gateway';
 import { BackendRepository } from '../repositories/backend.repository';
-import { ModuleConfigRepository } from '../repositories/moduleConfig.repository';
 import { TelemetryService } from './telemetry.service';
 
 @Injectable()
 export class BackendService {
   constructor(
-    private readonly moduleConfig: ModuleConfigRepository,
     private readonly repository: BackendRepository,
     private readonly events: EventsGateway,
     private readonly telemetry: TelemetryService,
@@ -22,7 +20,7 @@ export class BackendService {
 
     const error = await Promise.all(
       backends.map((backend) =>
-        Backend.from(backend.configuration, this.moduleConfig.get())
+        Backend.from(backend.configuration)
           .checkOnline()
           .then(() => void 0)
           .catch((error) => error),

packages/yucca-sdk/orchestration-api/src/services/repository.service.ts

@@ -4,7 +4,6 @@ import { randomUUID } from 'node:crypto';
 import { type WriteStream } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { Observable } from 'rxjs';
-import { Backend } from '../backends/backend';
 import { FilesystemListingRequestDto, FilesystemListingResponseDto } from '../dto/filesystem.dto';
 import {
   ListSnapshotsResponseDto,
@@ -86,9 +85,7 @@ export class RepositoryService {
       backendId = backends[0].id;
     }
 
-    const { configuration } = await this.backend.getBackend(backendId);
+    const { backend, configuration } = await this.backend.getBackend(backendId);
-    const backend = Backend.from(configuration, this.moduleConfig.get());
-
     const { repository: remote } = await backend.createRepository(dto);
     const id = randomUUID();
 
@@ -144,8 +141,7 @@ export class RepositoryService {
     const backendsById = Object.fromEntries(backends.map((backend) => [backend.id, backend]));
     const remoteRepositories: Record<string, Record<string, RepositoryWithMetricsDto>> = {};
 
-    for (const { id: backendId, configuration } of backends) {
+    for (const { id: backendId, backend } of backends) {
-      const backend = Backend.from(configuration, this.moduleConfig.get());
       remoteRepositories[backendId] = {};
 
       try {
@@ -289,14 +285,13 @@ export class RepositoryService {
       remoteId = localRepository.remoteId;
     }
 
-    const backend = await this.backend.getBackend(backendId);
+    const { backend, configuration } = await this.backend.getBackend(backendId);
-    const backendInstance = Backend.from(backend.configuration, this.moduleConfig.get());
 
     let remote;
     if (dto.name) {
-      ({ repository: remote } = await backendInstance.updateRepository(remoteId, dto));
+      ({ repository: remote } = await backend.updateRepository(remoteId, dto));
     } else {
-      ({ repository: remote } = await backendInstance.getRepository(remoteId));
+      ({ repository: remote } = await backend.getRepository(remoteId));
     }
 
     if (dto.paths) {
@@ -330,7 +325,7 @@ export class RepositoryService {
       backends: {
         primary: {
           id: backendId,
-          type: backend.configuration.type,
+          type: configuration.type,
           online: true,
         },
         secondary: [],
@@ -382,9 +377,8 @@ export class RepositoryService {
       ({ backendId, remoteId } = repository);
     }
 
-    const backend = await this.backend.getBackend(backendId);
+    const { backend } = await this.backend.getBackend(backendId);
-    const backendInstance = Backend.from(backend.configuration, this.moduleConfig.get());
+    const endpoint = await backend.getResticEndpoint(remoteId);
-    const endpoint = await backendInstance.getResticEndpoint(remoteId);
 
     const key = await this.config.deriveEncryptionKey(`repository-${remoteId}`);
 
@@ -424,8 +418,7 @@ export class RepositoryService {
 
       if (metrics.sizeBytes) {
         const { backendId, remoteId } = await this.repository.get(id);
-        const { configuration } = await this.backend.getBackend(backendId);
+        const { backend } = await this.backend.getBackend(backendId);
-        const backend = Backend.from(configuration, this.moduleConfig.get());
 
         if (backend.isMetricsCapable()) {
           await backend.submitMetricRepositorySize(remoteId, metrics.sizeBytes);
@@ -458,9 +451,7 @@ export class RepositoryService {
     });
 
     const { backendId, remoteId } = await this.repository.get(id);
-    const { configuration } = await this.backend.getBackend(backendId);
+    const { backend } = await this.backend.getBackend(backendId);
-    const backend = Backend.from(configuration, this.moduleConfig.get());
-
     const { endpoint, key } = await this.getResticParameters(id);
 
     const paths = await this.repositoryPath.get(id);
@@ -667,8 +658,7 @@ export class RepositoryService {
     });
 
     try {
-      const { configuration } = await this.backend.getBackend(backendId);
+      const { configuration, backend } = await this.backend.getBackend(backendId);
-      const backend = Backend.from(configuration, this.moduleConfig.get());
       const { repository: remote } = await backend.getRepository(id);
       const localId = randomUUID();
 
@@ -734,8 +724,7 @@ export class RepositoryService {
     id: string,
     dto: RepositoryPrimaryBackendReconfigureRequestDto,
   ): Promise<RepositoryCreateResponseDto> {
-    const { configuration } = await this.backend.getBackend(dto.backendId);
+    const { backend, configuration } = await this.backend.getBackend(dto.backendId);
-    const backend = Backend.from(configuration, this.moduleConfig.get());
 
     const { repository: remote } = await backend.createRepository({
       name: 'Restored Repository',

packages/yucca-sdk/orchestration-api/src/services/telemetry.service.ts

@@ -1,9 +1,8 @@
 import { Injectable } from '@nestjs/common';
 import { Backend } from '../backends/backend';
-import { YUCCA_PRODUCTION_UUID } from '../const';
+import { REPOSITORY_DEFAULT_CLOUD_UUID } from '../const';
 import { BackendRepository } from '../repositories/backend.repository';
 import { ConfigRepository } from '../repositories/config.repository';
-import { ModuleConfigRepository } from '../repositories/moduleConfig.repository';
 
 function serializeErrors(data: object): object {
   return Object.fromEntries(
@@ -22,7 +21,6 @@ export class TelemetryService {
   constructor(
     private readonly config: ConfigRepository,
     private readonly backend: BackendRepository,
-    private readonly moduleConfig: ModuleConfigRepository,
   ) {}
 
   private async submitStructuredLogImpl(summary: string, data: object, force: boolean) {
@@ -34,9 +32,9 @@ export class TelemetryService {
     }
 
     void this.backend
-      .getBackend(YUCCA_PRODUCTION_UUID)
+      .getBackend(REPOSITORY_DEFAULT_CLOUD_UUID)
       .then(({ configuration }) => {
-        const backend = Backend.from(configuration, this.moduleConfig.get());
+        const backend = Backend.from(configuration);
         backend.submitStructuredLog(summary, {
           ...serializeErrors(data),
           version,

tf/deployment/dev/dns/records.auto.tfvars

@@ -21,4 +21,11 @@ records = {
     values  = ["10.10.10.90", "10.10.10.91", "10.10.10.92"]
     comment = "Sietch RGW S3 virtual-hosted buckets (tf/deployment/dev/dns)"
   }
+  # move me to somewhere sensible!
+  "futo.cloud" = {
+    type    = "A"
+    values  = ["45.144.160.215"] # homelab
+    proxied = true
+    comment = "Website (just serving .well-known/yucca.json)"
+  }
 }