fix(yucca sdk): surface the state database path and guard master key generation (#653)

Co-authored-by: izzy <me@insrt.uk>

packages/yucca-sdk/orchestration-api/src/enum.ts

@@ -24,6 +24,7 @@ export enum ConfigurationKey {
   SkippedOnboardingExtraConfig = 'skipped-onboarding-extra-config',
   ResticOptionRestConnections = 'restic-o-rest-connections',
   SessionSecret = 'session-secret',
+  StatePath = 'state-path',
 }
 
 export enum MetadataKey {

packages/yucca-sdk/orchestration-api/src/orchestrationApi.module.ts

@@ -4,7 +4,7 @@ import { EventEmitterModule } from '@nestjs/event-emitter';
 import { JwtModule } from '@nestjs/jwt';
 import { ScheduleModule } from '@nestjs/schedule';
 import Database from 'better-sqlite3';
-import { SqliteDialect } from 'kysely';
+import { type SqliteDatabase, SqliteDialect } from 'kysely';
 import { KyselyModule } from 'nestjs-kysely';
 import { existsSync, mkdirSync } from 'node:fs';
 import { homedir } from 'node:os';
@@ -103,6 +103,27 @@ export const services = [
   YuccaService,
 ];
 
+export function openStateDatabase(
+  config: Pick<ModuleConfig, 'statePath'>,
+  logger = LoggingRepository.create('State'),
+): SqliteDatabase {
+  mkdirSync(config.statePath, { recursive: true });
+
+  const databasePath = resolve(config.statePath, 'state.sqlite3');
+
+  if (existsSync(databasePath)) {
+    logger.log(`Opened existing state database at ${databasePath}`);
+  } else {
+    logger.warn(`No state database at ${databasePath}, creating a new one`);
+  }
+
+  const database = new Database(databasePath);
+  database.pragma('journal_mode = WAL');
+  database.pragma('wal_autocheckpoint = 1');
+
+  return database;
+}
+
 export interface OrchestrationApiModuleAsyncOptions extends Pick<ModuleMetadata, 'imports'> {
   inject?: FactoryProvider['inject'];
   useFactory: (...args: any[]) => Promise<Partial<ModuleConfig>> | Partial<ModuleConfig>;
@@ -144,19 +165,9 @@ export class OrchestrationApiModule {
           namespace: 'orchestrator',
           imports: [configModule],
           inject: [ModuleConfigProvider],
-          useFactory: (config: ModuleConfig) => {
-            if (!existsSync(config.statePath)) {
-              mkdirSync(config.statePath, { recursive: true });
-            }
-
-            const database = new Database(resolve(config.statePath, 'state.sqlite3'));
-            database.pragma('journal_mode = WAL');
-            database.pragma('wal_autocheckpoint = 1');
-
-            return {
-              dialect: new SqliteDialect({ database }),
-            };
-          },
+          useFactory: (config: ModuleConfig) => ({
+            dialect: new SqliteDialect({ database: openStateDatabase(config) }),
+          }),
         }),
         EventEmitterModule.forRoot(),
         JwtModule.register({}),

packages/yucca-sdk/orchestration-api/src/repositories/config.repository.spec.ts

@@ -1,6 +1,7 @@
 import { availableParallelism } from 'node:os';
 import { yuccaWellKnown } from '../wellKnown';
 import { ConfigRepository } from './config.repository';
+import { LoggingRepository } from './logging.repository';
 
 describe(ConfigRepository.name, () => {
   const db = {
@@ -20,7 +21,7 @@ describe(ConfigRepository.name, () => {
   it('resolves options for a placed repository', async () => {
     const connections = jest.spyOn(yuccaWellKnown, 'getConnections').mockResolvedValue(7);
     const packSize = jest.spyOn(yuccaWellKnown, 'getPackSizeMib').mockResolvedValue(64);
-    const repository = new ConfigRepository(db as never);
+    const repository = new ConfigRepository(db as never, { statePath: '/state' }, LoggingRepository.create());
 
     await expect(
       repository.getResticOptions({ siteCode: 'father', storageClusterCode: 'father-spice' }),
@@ -33,7 +34,7 @@ describe(ConfigRepository.name, () => {
   it('falls back to global config and core count without placement', async () => {
     const connections = jest.spyOn(yuccaWellKnown, 'getConnections').mockResolvedValue();
     const packSize = jest.spyOn(yuccaWellKnown, 'getPackSizeMib').mockResolvedValue();
-    const repository = new ConfigRepository(db as never);
+    const repository = new ConfigRepository(db as never, { statePath: '/state' }, LoggingRepository.create());
 
     await expect(repository.getResticOptions({ siteCode: null, storageClusterCode: null })).resolves.toEqual({
       connections: availableParallelism(),

packages/yucca-sdk/orchestration-api/src/repositories/config.repository.ts

@@ -1,25 +1,43 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { Kysely } from 'kysely';
 import { InjectKysely } from 'nestjs-kysely';
 import { randomBytes } from 'node:crypto';
 import { availableParallelism } from 'node:os';
+import { resolve } from 'node:path';
 import { ConfigurationKey } from '../enum';
+import { type ModuleConfig, ModuleConfigProvider } from '../moduleConfig';
 import { DB } from '../schema';
 import { yuccaWellKnown } from '../wellKnown';
+import { LoggingRepository } from './logging.repository';
 
 export type ResticPlacement = { siteCode: string | null; storageClusterCode: string | null };
 
 @Injectable()
 export class ConfigRepository {
-  constructor(@InjectKysely('orchestrator') private db: Kysely<DB>) {}
+  constructor(
+    @InjectKysely('orchestrator') private db: Kysely<DB>,
+    @Inject(ModuleConfigProvider) private moduleConfig: ModuleConfig,
+    private readonly logger: LoggingRepository,
+  ) {
+    this.logger.setContext(ConfigRepository.name);
+  }
 
   async bootstrap() {
+    const statePath = resolve(this.moduleConfig.statePath);
     const hasKey = await this.hasEncryptionKey();
 
     if (!hasKey) {
+      await this.assertStateIsEmpty(statePath);
+
+      this.logger.warn(
+        `Generating a new master encryption key for the state in ${statePath}. Backups written with a previous key cannot be decrypted with this one.`,
+      );
+
       await this.set(ConfigurationKey.EncryptionKey, randomBytes(32).toString('hex'));
     }
 
+    await this.recordStatePath(statePath);
+
     const hasSecret = await this.hasSessionSecret();
 
     if (!hasSecret) {
@@ -27,6 +45,42 @@ export class ConfigRepository {
     }
   }
 
+  private async assertStateIsEmpty(statePath: string) {
+    const [{ backends }] = await this.db
+      .selectFrom('backends')
+      .select((eb) => eb.fn.countAll<number>().as('backends'))
+      .execute();
+
+    const [{ repositories }] = await this.db
+      .selectFrom('repositories')
+      .select((eb) => eb.fn.countAll<number>().as('repositories'))
+      .execute();
+
+    if (backends === 0 && repositories === 0) {
+      return;
+    }
+
+    throw new Error(
+      `The state in ${statePath} has ${backends} backend(s) and ${repositories} repository(ies) but no master encryption key. Refusing to generate one, as that would make those backups undecryptable. Restore the original state database.`,
+    );
+  }
+
+  private async recordStatePath(statePath: string) {
+    const previous = await this.getOptional(ConfigurationKey.StatePath);
+
+    if (previous === statePath) {
+      return;
+    }
+
+    if (previous) {
+      this.logger.warn(
+        `The state database was created in ${previous} but is being opened from ${statePath}. If the original directory is still populated, two installations are now diverging.`,
+      );
+    }
+
+    await this.set(ConfigurationKey.StatePath, statePath);
+  }
+
   private async set(key: ConfigurationKey, value: string) {
     await this.db
       .insertInto('config')

packages/yucca-sdk/orchestration-api/test/statePath.integration-spec.ts

@@ -0,0 +1,137 @@
+import { Logger } from '@nestjs/common';
+import { Test } from '@nestjs/testing';
+import Database from 'better-sqlite3';
+import { SqliteDialect } from 'kysely';
+import { KyselyModule } from 'nestjs-kysely';
+import { copyFile, mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ModuleConfigProvider } from 'src/moduleConfig';
+import { openStateDatabase } from 'src/orchestrationApi.module';
+import { ConfigRepository } from 'src/repositories/config.repository';
+import { DatabaseRepository } from 'src/repositories/database.repository';
+import { LoggingRepository } from 'src/repositories/logging.repository';
+import { StorageRepository } from 'src/repositories/storage.repository';
+
+interface Boot {
+  config: ConfigRepository;
+  database: InstanceType<typeof Database>;
+  close: () => Promise<void>;
+}
+
+async function boot(statePath: string): Promise<Boot> {
+  const database = new Database(join(statePath, 'state.sqlite3'));
+
+  const moduleFixture = await Test.createTestingModule({
+    imports: [KyselyModule.forRoot([{ namespace: 'orchestrator', dialect: new SqliteDialect({ database }) }])],
+    providers: [
+      { provide: ModuleConfigProvider, useValue: { statePath } },
+      LoggingRepository,
+      StorageRepository,
+      DatabaseRepository,
+      ConfigRepository,
+    ],
+  }).compile();
+
+  await moduleFixture.get(DatabaseRepository).runMigrations();
+
+  return {
+    config: moduleFixture.get(ConfigRepository),
+    database,
+    close: async () => {
+      await moduleFixture.close();
+      if (database.open) {
+        database.close();
+      }
+    },
+  };
+}
+
+function messages(spy: jest.SpyInstance): string[] {
+  return spy.mock.calls.map(([message]) => String(message));
+}
+
+let statePath: string;
+let warn: jest.SpyInstance;
+let log: jest.SpyInstance;
+
+beforeEach(async () => {
+  statePath = await mkdtemp(join(tmpdir(), 'yucca-state-'));
+  warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
+  log = jest.spyOn(Logger.prototype, 'log').mockImplementation(() => {});
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe('State location', () => {
+  it('announces that a new state database is being created, then that it is reopened', () => {
+    const first = openStateDatabase({ statePath });
+    first.close();
+
+    expect(messages(warn).join('\n')).toContain(join(statePath, 'state.sqlite3'));
+    expect(messages(warn).join('\n')).toMatch(/creating a new one/i);
+
+    warn.mockClear();
+    const second = openStateDatabase({ statePath });
+    second.close();
+
+    expect(warn).not.toHaveBeenCalled();
+    expect(messages(log).join('\n')).toMatch(/opened existing state database/i);
+  });
+
+  it('warns loudly when it generates a master encryption key', async () => {
+    const first = await boot(statePath);
+    await first.config.bootstrap();
+    const key = await first.config.getMasterEncryptionKey();
+    await first.close();
+
+    expect(messages(warn).join('\n')).toMatch(/master encryption key/i);
+    expect(messages(warn).join('\n')).toContain(statePath);
+
+    warn.mockClear();
+
+    const second = await boot(statePath);
+    await second.config.bootstrap();
+
+    await expect(second.config.getMasterEncryptionKey()).resolves.toEqual(key);
+    expect(warn).not.toHaveBeenCalled();
+
+    await second.close();
+  });
+
+  it('warns when the state database is opened from a different path than it was created at', async () => {
+    const first = await boot(statePath);
+    await first.config.bootstrap();
+    await first.close();
+
+    const movedPath = await mkdtemp(join(tmpdir(), 'yucca-state-moved-'));
+    await copyFile(join(statePath, 'state.sqlite3'), join(movedPath, 'state.sqlite3'));
+    warn.mockClear();
+
+    const second = await boot(movedPath);
+    await second.config.bootstrap();
+    await second.close();
+
+    const warnings = messages(warn).join('\n');
+    expect(warnings).toContain(statePath);
+    expect(warnings).toContain(movedPath);
+  });
+
+  it('refuses to generate a new master encryption key when repositories already exist', async () => {
+    const first = await boot(statePath);
+    await first.config.bootstrap();
+    first.database.prepare('INSERT INTO backends (id, configuration) VALUES (?, ?)').run('backend', '{}');
+    first.database.prepare('INSERT INTO repositories (id, backendId) VALUES (?, ?)').run('repository', 'backend');
+    first.database.prepare("DELETE FROM config WHERE key = 'encryption-key'").run();
+    await first.close();
+
+    const second = await boot(statePath);
+
+    await expect(second.config.bootstrap()).rejects.toThrow(/encryption key/i);
+    await expect(second.config.hasEncryptionKey()).resolves.toBe(false);
+
+    await second.close();
+  });
+});