diff --git a/packages/yucca-sdk/orchestration-api/src/backends/backend.ts b/packages/yucca-sdk/orchestration-api/src/backends/backend.ts index 67ac0409..7e252c07 100644 --- a/packages/yucca-sdk/orchestration-api/src/backends/backend.ts +++ b/packages/yucca-sdk/orchestration-api/src/backends/backend.ts @@ -7,7 +7,6 @@ import { RepositoryUpdateResponseDto, } from '@futo-org/backups-api-client'; import { BackendType } from '../enum'; -import { ModuleConfig } from '../moduleConfig'; import { BackendConfiguration } from '../schema/tables/backend.table'; export abstract class Backend { @@ -30,13 +29,10 @@ export abstract class Backend { abstract submitMetricRepositorySize(id: string, size: number): Promise; abstract submitStructuredLog(summary: string, data: object): void; - static from(configuration: BackendConfiguration, moduleConfig: ModuleConfig) { + static from(configuration: BackendConfiguration) { switch (configuration.type) { case BackendType.Yucca: { - return new YuccaBackend({ - url: moduleConfig.yuccaProductionApi, - ...configuration, - }); + return new YuccaBackend(configuration); } case BackendType.Local: { return new LocalBackend(configuration); diff --git a/packages/yucca-sdk/orchestration-api/src/backends/yucca.backend.ts b/packages/yucca-sdk/orchestration-api/src/backends/yucca.backend.ts index bf4bae47..531178f5 100644 --- a/packages/yucca-sdk/orchestration-api/src/backends/yucca.backend.ts +++ b/packages/yucca-sdk/orchestration-api/src/backends/yucca.backend.ts @@ -13,18 +13,49 @@ import { submitStructuredLog, updateRepository, } from '@futo-org/backups-api-client'; +import { YUCCA_WELL_KNOWN } from '../const'; import { BackendType, CookieName } from '../enum'; import { BackendConfiguration } from '../schema/tables/backend.table'; import { Backend } from './backend'; +type WellKnown = { + backends: Record; + defaultBackend: string; +}; + +class YuccaWellKnown { + private data?: WellKnown; + + async get() { + this.data ??= await fetch(YUCCA_WELL_KNOWN).then((response) => response.json()); + return this.data! as WellKnown; + } + + async getBaseUrlById(id: string) { + const { backends } = await this.get(); + return backends[id].api; + } + + async getBaseUrl() { + const { backends, defaultBackend } = await this.get(); + return backends[defaultBackend].api; + } +} + +export const yuccaWellKnown = new YuccaWellKnown(); + export class YuccaBackend extends Backend { - constructor(protected readonly configuration: BackendConfiguration & { type: BackendType.Yucca; url: string }) { + constructor(protected readonly configuration: BackendConfiguration & { type: BackendType.Yucca; url?: string }) { super(configuration); } - private get requestOptions() { + private async getRequestOptions() { return { - baseUrl: this.configuration.url, + baseUrl: + this.configuration.url ?? + (this.configuration.uuid + ? await yuccaWellKnown.getBaseUrlById(this.configuration.uuid) + : await yuccaWellKnown.getBaseUrl()), headers: { cookie: `${CookieName.YuccaAccessToken}=${this.configuration.accessToken}`, }, @@ -40,60 +71,56 @@ export class YuccaBackend extends Backend { } async checkOnline() { - await getAuth(this.requestOptions); + await getAuth(await this.getRequestOptions()); } - createRepository(dto: RepositoryCreateRequestDto) { - return createRepository(dto, this.requestOptions); + async createRepository(dto: RepositoryCreateRequestDto) { + return await createRepository(dto, await this.getRequestOptions()); } - updateRepository(id: string, dto: RepositoryUpdateRequestDto) { - return updateRepository(id, dto, this.requestOptions); + async updateRepository(id: string, dto: RepositoryUpdateRequestDto) { + return await updateRepository(id, dto, await this.getRequestOptions()); } - getRepository(id: string) { - return getRepository(id, this.requestOptions); + async getRepository(id: string) { + return await getRepository(id, await this.getRequestOptions()); } - getRepositories() { - return getRepositories(this.requestOptions); + async getRepositories() { + return await getRepositories(await this.getRequestOptions()); } - deleteRepository(id: string) { - return deleteRepository(id, this.requestOptions); + async deleteRepository(id: string) { + return await deleteRepository(id, await this.getRequestOptions()); } async getResticEndpoint(id: string) { - const { url } = await createResticUrl(id, this.requestOptions); + const { url } = await createResticUrl(id, await this.getRequestOptions()); return url; } - submitMetricBackupStart(id: string): Promise { - return submitMetricBackupStart(id, this.requestOptions); + async submitMetricBackupStart(id: string): Promise { + return await submitMetricBackupStart(id, await this.getRequestOptions()); } - submitMetricBackupEnd(id: string, success: boolean, durationMs: number): Promise { - return submitMetricBackupEnd( + async submitMetricBackupEnd(id: string, success: boolean, durationMs: number): Promise { + return await submitMetricBackupEnd( id, { durationMs, success, }, - this.requestOptions, + await this.getRequestOptions(), ); } - submitMetricRepositorySize(id: string, sizeBytes: number): Promise { - return submitMetricRepositorySize(id, { sizeBytes }, this.requestOptions); + async submitMetricRepositorySize(id: string, sizeBytes: number): Promise { + return await submitMetricRepositorySize(id, { sizeBytes }, await this.getRequestOptions()); } submitStructuredLog(summary: string, data: object) { - void submitStructuredLog( - { - summary, - data, - }, - this.requestOptions, - ).catch((error) => console.error('Failed to submit log', error)); + void this.getRequestOptions() + .then((requestOptions) => submitStructuredLog({ summary, data }, requestOptions)) + .catch((error) => console.error('Failed to submit log', error)); } } diff --git a/packages/yucca-sdk/orchestration-api/src/const.ts b/packages/yucca-sdk/orchestration-api/src/const.ts index 9dd0026a..e3167dac 100644 --- a/packages/yucca-sdk/orchestration-api/src/const.ts +++ b/packages/yucca-sdk/orchestration-api/src/const.ts @@ -1,2 +1,3 @@ export const ORCHESTRATION_PORT = 22_676; -export const YUCCA_PRODUCTION_UUID = 'd0368cdd-39ae-40e1-91d6-d81815c65c7e'; +export const REPOSITORY_DEFAULT_CLOUD_UUID = 'd0368cdd-39ae-40e1-91d6-d81815c65c7e'; +export const YUCCA_WELL_KNOWN = 'https://futo.cloud/.well-known/yucca.json'; diff --git a/packages/yucca-sdk/orchestration-api/src/moduleConfig.ts b/packages/yucca-sdk/orchestration-api/src/moduleConfig.ts index dd31fbb8..1bb5e28e 100644 --- a/packages/yucca-sdk/orchestration-api/src/moduleConfig.ts +++ b/packages/yucca-sdk/orchestration-api/src/moduleConfig.ts @@ -13,7 +13,7 @@ export type ImmichIntegration = { export type ModuleConfig = { statePath: string; - yuccaProductionApi: string; + yuccaProductionApi?: string; externalBaseUrl?: string; requireWsAuth?: boolean; requireLock?: boolean; diff --git a/packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts b/packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts index fe20e3bb..39f43895 100644 --- a/packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts +++ b/packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts @@ -95,13 +95,9 @@ export const services = [ @Module({}) export class OrchestrationApiModule { - static forRoot(config: Partial & Pick): DynamicModule { + static forRoot(config: Partial): DynamicModule { config.statePath ??= resolve(homedir(), '.yucca'); - if (!config.yuccaProductionApi) { - throw new Error('config.yuccaProductionApi is missing'); - } - return { module: OrchestrationApiModule, imports: [ diff --git a/packages/yucca-sdk/orchestration-api/src/repositories/backend.repository.ts b/packages/yucca-sdk/orchestration-api/src/repositories/backend.repository.ts index c001a099..937188c3 100644 --- a/packages/yucca-sdk/orchestration-api/src/repositories/backend.repository.ts +++ b/packages/yucca-sdk/orchestration-api/src/repositories/backend.repository.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; +import { Backend } from '../backends/backend'; import { DB } from '../schema'; import { BackendConfiguration } from '../schema/tables/backend.table'; @@ -22,10 +23,12 @@ export class BackendRepository { async getBackends() { const backends = await this.db.selectFrom('backends').selectAll('backends').execute(); - return backends.map(({ id, configuration }) => ({ - id, - configuration: JSON.parse(configuration) as BackendConfiguration, - })); + return backends + .map(({ id, configuration }) => ({ + id, + configuration: JSON.parse(configuration) as BackendConfiguration, + })) + .map(({ id, configuration }) => ({ id, configuration, backend: Backend.from(configuration) })); } async getBackend(id: string) { @@ -35,9 +38,12 @@ export class BackendRepository { .where('id', '=', id) .executeTakeFirstOrThrow(); + const configuration = JSON.parse(backend.configuration) as BackendConfiguration; + return { id: backend.id, - configuration: JSON.parse(backend.configuration) as BackendConfiguration, + configuration, + backend: Backend.from(configuration), }; } } diff --git a/packages/yucca-sdk/orchestration-api/src/schema/tables/backend.table.ts b/packages/yucca-sdk/orchestration-api/src/schema/tables/backend.table.ts index 96ba50d8..7556daa6 100644 --- a/packages/yucca-sdk/orchestration-api/src/schema/tables/backend.table.ts +++ b/packages/yucca-sdk/orchestration-api/src/schema/tables/backend.table.ts @@ -11,6 +11,7 @@ export type BackendConfiguration = */ type: BackendType.Yucca; url?: string; + uuid?: string; accessToken: string; } | { diff --git a/packages/yucca-sdk/orchestration-api/src/services/auth.service.ts b/packages/yucca-sdk/orchestration-api/src/services/auth.service.ts index 5fe3bbba..70f8534e 100644 --- a/packages/yucca-sdk/orchestration-api/src/services/auth.service.ts +++ b/packages/yucca-sdk/orchestration-api/src/services/auth.service.ts @@ -1,6 +1,7 @@ import { Injectable, InternalServerErrorException } from '@nestjs/common'; import { createEventSource, EventSourceClient } from 'eventsource-client'; -import { YUCCA_PRODUCTION_UUID } from '../const'; +import { yuccaWellKnown } from '../backends/yucca.backend'; +import { REPOSITORY_DEFAULT_CLOUD_UUID } from '../const'; import { BackendType } from '../enum'; import { EventsGateway } from '../events/events.gateway'; import { BackendRepository } from '../repositories/backend.repository'; @@ -18,25 +19,26 @@ export class AuthService { readonly telemetry: TelemetryService, ) {} - private async waitForDeviceFlow(events: EventSourceClient) { + private async waitForDeviceFlow(events: EventSourceClient, url?: string) { for await (const { data } of events) { const { type, accessToken } = JSON.parse(data); switch (type) { case 'SUCCESS': { - await this.backend.updateBackend(YUCCA_PRODUCTION_UUID, { + await this.backend.updateBackend(REPOSITORY_DEFAULT_CLOUD_UUID, { type: BackendType.Yucca, accessToken, + url, }); this.telemetry.submitStructuredLog('Connected FUTO Backups backend', { - backendId: YUCCA_PRODUCTION_UUID, + backendId: REPOSITORY_DEFAULT_CLOUD_UUID, }); this.events.publish({ type: 'BackendCreate', backend: { - id: YUCCA_PRODUCTION_UUID, + id: REPOSITORY_DEFAULT_CLOUD_UUID, type: BackendType.Yucca, description: 'FUTO Backups', isOnline: true, @@ -61,15 +63,18 @@ export class AuthService { } async oidcDeviceFlow(): Promise<{ userCode: string; verificationUri: string }> { + const overrideEndpoint = this.moduleConfig.get().yuccaProductionApi; + const endpoint = overrideEndpoint ?? (await yuccaWellKnown.getBaseUrl()); + const events: EventSourceClient = createEventSource({ - url: new URL('/api/auth/oidc/device', this.moduleConfig.get().yuccaProductionApi), + url: new URL('/api/auth/oidc/device', endpoint), onDisconnect: () => events.close(), }); for await (const { data } of events) { const { userCode, verificationUri } = JSON.parse(data); - void this.waitForDeviceFlow(events).catch(() => {}); + void this.waitForDeviceFlow(events, overrideEndpoint).catch(() => {}); return { userCode, diff --git a/packages/yucca-sdk/orchestration-api/src/services/backend.service.ts b/packages/yucca-sdk/orchestration-api/src/services/backend.service.ts index b158ad85..02ac04aa 100644 --- a/packages/yucca-sdk/orchestration-api/src/services/backend.service.ts +++ b/packages/yucca-sdk/orchestration-api/src/services/backend.service.ts @@ -5,13 +5,11 @@ import { BackendResponseDto, BackendsResponseDto, CreateLocalBackendRequestDto } import { BackendType } from '../enum'; import { EventsGateway } from '../events/events.gateway'; import { BackendRepository } from '../repositories/backend.repository'; -import { ModuleConfigRepository } from '../repositories/moduleConfig.repository'; import { TelemetryService } from './telemetry.service'; @Injectable() export class BackendService { constructor( - private readonly moduleConfig: ModuleConfigRepository, private readonly repository: BackendRepository, private readonly events: EventsGateway, private readonly telemetry: TelemetryService, @@ -22,7 +20,7 @@ export class BackendService { const error = await Promise.all( backends.map((backend) => - Backend.from(backend.configuration, this.moduleConfig.get()) + Backend.from(backend.configuration) .checkOnline() .then(() => void 0) .catch((error) => error), diff --git a/packages/yucca-sdk/orchestration-api/src/services/repository.service.ts b/packages/yucca-sdk/orchestration-api/src/services/repository.service.ts index 2f928c5e..f1a9c25f 100644 --- a/packages/yucca-sdk/orchestration-api/src/services/repository.service.ts +++ b/packages/yucca-sdk/orchestration-api/src/services/repository.service.ts @@ -4,7 +4,6 @@ import { randomUUID } from 'node:crypto'; import { type WriteStream } from 'node:fs'; import { dirname, join } from 'node:path'; import { Observable } from 'rxjs'; -import { Backend } from '../backends/backend'; import { FilesystemListingRequestDto, FilesystemListingResponseDto } from '../dto/filesystem.dto'; import { ListSnapshotsResponseDto, @@ -86,9 +85,7 @@ export class RepositoryService { backendId = backends[0].id; } - const { configuration } = await this.backend.getBackend(backendId); - const backend = Backend.from(configuration, this.moduleConfig.get()); - + const { backend, configuration } = await this.backend.getBackend(backendId); const { repository: remote } = await backend.createRepository(dto); const id = randomUUID(); @@ -144,8 +141,7 @@ export class RepositoryService { const backendsById = Object.fromEntries(backends.map((backend) => [backend.id, backend])); const remoteRepositories: Record> = {}; - for (const { id: backendId, configuration } of backends) { - const backend = Backend.from(configuration, this.moduleConfig.get()); + for (const { id: backendId, backend } of backends) { remoteRepositories[backendId] = {}; try { @@ -289,14 +285,13 @@ export class RepositoryService { remoteId = localRepository.remoteId; } - const backend = await this.backend.getBackend(backendId); - const backendInstance = Backend.from(backend.configuration, this.moduleConfig.get()); + const { backend, configuration } = await this.backend.getBackend(backendId); let remote; if (dto.name) { - ({ repository: remote } = await backendInstance.updateRepository(remoteId, dto)); + ({ repository: remote } = await backend.updateRepository(remoteId, dto)); } else { - ({ repository: remote } = await backendInstance.getRepository(remoteId)); + ({ repository: remote } = await backend.getRepository(remoteId)); } if (dto.paths) { @@ -330,7 +325,7 @@ export class RepositoryService { backends: { primary: { id: backendId, - type: backend.configuration.type, + type: configuration.type, online: true, }, secondary: [], @@ -382,9 +377,8 @@ export class RepositoryService { ({ backendId, remoteId } = repository); } - const backend = await this.backend.getBackend(backendId); - const backendInstance = Backend.from(backend.configuration, this.moduleConfig.get()); - const endpoint = await backendInstance.getResticEndpoint(remoteId); + const { backend } = await this.backend.getBackend(backendId); + const endpoint = await backend.getResticEndpoint(remoteId); const key = await this.config.deriveEncryptionKey(`repository-${remoteId}`); @@ -424,8 +418,7 @@ export class RepositoryService { if (metrics.sizeBytes) { const { backendId, remoteId } = await this.repository.get(id); - const { configuration } = await this.backend.getBackend(backendId); - const backend = Backend.from(configuration, this.moduleConfig.get()); + const { backend } = await this.backend.getBackend(backendId); if (backend.isMetricsCapable()) { await backend.submitMetricRepositorySize(remoteId, metrics.sizeBytes); @@ -458,9 +451,7 @@ export class RepositoryService { }); const { backendId, remoteId } = await this.repository.get(id); - const { configuration } = await this.backend.getBackend(backendId); - const backend = Backend.from(configuration, this.moduleConfig.get()); - + const { backend } = await this.backend.getBackend(backendId); const { endpoint, key } = await this.getResticParameters(id); const paths = await this.repositoryPath.get(id); @@ -667,8 +658,7 @@ export class RepositoryService { }); try { - const { configuration } = await this.backend.getBackend(backendId); - const backend = Backend.from(configuration, this.moduleConfig.get()); + const { configuration, backend } = await this.backend.getBackend(backendId); const { repository: remote } = await backend.getRepository(id); const localId = randomUUID(); @@ -734,8 +724,7 @@ export class RepositoryService { id: string, dto: RepositoryPrimaryBackendReconfigureRequestDto, ): Promise { - const { configuration } = await this.backend.getBackend(dto.backendId); - const backend = Backend.from(configuration, this.moduleConfig.get()); + const { backend, configuration } = await this.backend.getBackend(dto.backendId); const { repository: remote } = await backend.createRepository({ name: 'Restored Repository', diff --git a/packages/yucca-sdk/orchestration-api/src/services/telemetry.service.ts b/packages/yucca-sdk/orchestration-api/src/services/telemetry.service.ts index c435c903..eff2c47a 100644 --- a/packages/yucca-sdk/orchestration-api/src/services/telemetry.service.ts +++ b/packages/yucca-sdk/orchestration-api/src/services/telemetry.service.ts @@ -1,9 +1,8 @@ import { Injectable } from '@nestjs/common'; import { Backend } from '../backends/backend'; -import { YUCCA_PRODUCTION_UUID } from '../const'; +import { REPOSITORY_DEFAULT_CLOUD_UUID } from '../const'; import { BackendRepository } from '../repositories/backend.repository'; import { ConfigRepository } from '../repositories/config.repository'; -import { ModuleConfigRepository } from '../repositories/moduleConfig.repository'; function serializeErrors(data: object): object { return Object.fromEntries( @@ -22,7 +21,6 @@ export class TelemetryService { constructor( private readonly config: ConfigRepository, private readonly backend: BackendRepository, - private readonly moduleConfig: ModuleConfigRepository, ) {} private async submitStructuredLogImpl(summary: string, data: object, force: boolean) { @@ -34,9 +32,9 @@ export class TelemetryService { } void this.backend - .getBackend(YUCCA_PRODUCTION_UUID) + .getBackend(REPOSITORY_DEFAULT_CLOUD_UUID) .then(({ configuration }) => { - const backend = Backend.from(configuration, this.moduleConfig.get()); + const backend = Backend.from(configuration); backend.submitStructuredLog(summary, { ...serializeErrors(data), version, diff --git a/tf/deployment/dev/dns/records.auto.tfvars b/tf/deployment/dev/dns/records.auto.tfvars index 62e2fd1c..559a5e7e 100644 --- a/tf/deployment/dev/dns/records.auto.tfvars +++ b/tf/deployment/dev/dns/records.auto.tfvars @@ -21,4 +21,11 @@ records = { values = ["10.10.10.90", "10.10.10.91", "10.10.10.92"] comment = "Sietch RGW S3 virtual-hosted buckets (tf/deployment/dev/dns)" } + # move me to somewhere sensible! + "futo.cloud" = { + type = "A" + values = ["45.144.160.215"] # homelab + proxied = true + comment = "Website (just serving .well-known/yucca.json)" + } }