feat(sql-tools): maintain and verify a migration ORDER file (#688)

packages/sql-tools/README.md

@@ -7,3 +7,20 @@ Kysely-based tools and utilities for managing postgres schema.
 ```bash
 npm i @immich/sql-tools
 ```
+
+## Migration order file
+
+If an `ORDER` file exists in the migration folder, `migrations create`, `migrations generate`, and `migrations revert` keep it in sync: one migration name per line, in the (sorted) order the migrator will run them. The file gives concurrent migration PRs a merge conflict instead of silently landing out of order, and gives CI something cheap to verify. Every file in the folder except `ORDER` itself is treated as a migration — stray files will fail verification, deliberately.
+
+```bash
+# create or refresh the file (defaults to src/schema/migrations, override with --source-folder)
+sql-tools migrations sync-order
+
+# verify it matches the migration files on disk
+sql-tools migrations verify-order
+
+# additionally verify no entries were inserted or removed relative to a baseline (e.g. the merge base in CI)
+sql-tools migrations verify-order --append-only-from /tmp/base-order
+```
+
+Repositories that don't have an `ORDER` file are unaffected.

packages/sql-tools/src/bin/cli.ts

@@ -1,7 +1,9 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
 import { sql } from 'kysely';
-import { join } from 'node:path';
+import { readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { ORDER_FILENAME, parseOrder, readOrder, syncOrder, verifyOrder } from 'src/migration-order';
 import { Migrator } from 'src/migration';
 
 const withMigrator =
@@ -11,6 +13,9 @@ const withMigrator =
     async function (...args: any[]) {
       const command: Command = args.at(-1);
       const options = command.optsWithGlobals();
+      if (!options.url) {
+        throw new Error(`Missing required option '-u, --url <url>'`);
+      }
 
       const migrator = new Migrator({
         connectionParams: { connectionType: 'url', url: options.url },
@@ -24,7 +29,7 @@ const withMigrator =
 const program = new Command('sql-tools');
 
 program
-  .requiredOption('-u, --url <url>', 'Database connection url')
+  .option('-u, --url <url>', 'Database connection url')
   .option(
     '-f, --folder <migrationsFolder>',
     'Path to the runnable (compiled) migration files',
@@ -65,6 +70,37 @@ migrations
   )
   .action(withMigrator((migrator, _, [path]) => migrator.create(join(process.cwd(), path ?? 'src/Migration'), [], [])));
 
+migrations
+  .command('sync-order')
+  .description(`Regenerate the ${ORDER_FILENAME} file from the migration files on disk`)
+  .action((_: unknown, command: Command) => {
+    const folder = resolve(process.cwd(), command.optsWithGlobals().sourceFolder);
+    const { changed, next } = syncOrder(folder);
+    console.log(
+      changed
+        ? `Wrote ${join(folder, ORDER_FILENAME)} (${next.length} migrations)`
+        : `${ORDER_FILENAME} is already up to date (${next.length} migrations)`,
+    );
+  });
+
+migrations
+  .command('verify-order')
+  .description(`Verify the ${ORDER_FILENAME} file matches the migration files on disk`)
+  .option('--append-only-from <path>', `Baseline ${ORDER_FILENAME} file that must be a prefix of the current one`)
+  .action((options: { appendOnlyFrom?: string }, command: Command) => {
+    const folder = resolve(process.cwd(), command.optsWithGlobals().sourceFolder);
+    const appendOnlyFrom = options.appendOnlyFrom
+      ? parseOrder(readFileSync(options.appendOnlyFrom, 'utf8'))
+      : undefined;
+
+    const errors = verifyOrder(folder, { appendOnlyFrom });
+    if (errors.length > 0) {
+      throw new Error(errors.map((error) => `- ${error}`).join('\n'));
+    }
+
+    console.log(`${ORDER_FILENAME} is consistent (${readOrder(folder)?.length ?? 0} migrations)`);
+  });
+
 migrations
   .command('generate')
   .description('Generate a new migration file that contains the UP and DOWN queries to migrate the schema')

packages/sql-tools/src/index.ts

@@ -24,6 +24,7 @@ export * from 'src/decorators/trigger.decorator';
 export * from 'src/decorators/unique.decorator';
 export * from 'src/decorators/update-date-column.decorator';
 export * from 'src/migration';
+export * from 'src/migration-order';
 export * from 'src/naming/default.naming';
 export * from 'src/naming/naming.interface';
 export * from 'src/register-enum';

packages/sql-tools/src/migration-order.spec.ts

@@ -0,0 +1,87 @@
+import { computeOrder, ORDER_FILENAME, parseOrder, verifyOrderContent } from 'src/migration-order';
+import { describe, expect, it } from 'vitest';
+
+describe(ORDER_FILENAME, () => {
+  describe(computeOrder.name, () => {
+    it('should list migration names sorted and without extensions', () => {
+      expect(computeOrder(['2000-b.ts', '1000-a.ts'])).toEqual(['1000-a', '2000-b']);
+    });
+
+    it('should ignore the ORDER file', () => {
+      expect(computeOrder(['1000-a.ts', ORDER_FILENAME])).toEqual(['1000-a']);
+    });
+
+    it('should include stray files so they fail verification loudly', () => {
+      expect(computeOrder(['1000-a.ts', '.DS_Store', 'README.md'])).toEqual(['.DS_Store', '1000-a', 'README']);
+    });
+
+    it('should keep colliding names so verification fails loudly', () => {
+      expect(computeOrder(['1000-a.ts', '1000-a.js'])).toEqual(['1000-a', '1000-a']);
+    });
+  });
+
+  describe(parseOrder.name, () => {
+    it('should ignore blank lines and surrounding whitespace', () => {
+      expect(parseOrder('1000-a\n\n 2000-b \n')).toEqual(['1000-a', '2000-b']);
+    });
+  });
+
+  describe(verifyOrderContent.name, () => {
+    it('should pass when the content matches', () => {
+      expect(verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a', '2000-b'] })).toEqual([]);
+    });
+
+    it('should fail on duplicate names', () => {
+      expect(verifyOrderContent({ actual: ['1000-a', '1000-a'], expected: ['1000-a', '1000-a'] })).toEqual([
+        expect.stringContaining('Duplicate migration name "1000-a"'),
+        expect.stringContaining(`Duplicate ${ORDER_FILENAME} entry "1000-a"`),
+      ]);
+    });
+
+    it('should fail when a migration is missing from the file', () => {
+      expect(verifyOrderContent({ actual: ['1000-a'], expected: ['1000-a', '2000-b'] })).toEqual([
+        expect.stringContaining(`"2000-b" is missing from ${ORDER_FILENAME}`),
+      ]);
+    });
+
+    it('should fail when the file lists a migration that does not exist', () => {
+      expect(verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a'] })).toEqual([
+        expect.stringContaining('does not exist'),
+      ]);
+    });
+
+    it('should fail when the entries are out of order', () => {
+      expect(verifyOrderContent({ actual: ['2000-b', '1000-a'], expected: ['1000-a', '2000-b'] })).toEqual([
+        expect.stringContaining('out of order'),
+      ]);
+    });
+
+    it('should pass when the baseline is a prefix', () => {
+      const lists = { actual: ['1000-a', '2000-b'], expected: ['1000-a', '2000-b'] };
+      expect(verifyOrderContent({ ...lists, appendOnlyFrom: ['1000-a'] })).toEqual([]);
+      expect(verifyOrderContent({ ...lists, appendOnlyFrom: ['1000-a', '2000-b'] })).toEqual([]);
+    });
+
+    it('should fail when an entry was added before existing ones', () => {
+      expect(
+        verifyOrderContent({
+          actual: ['1500-a', '2000-b'],
+          expected: ['1500-a', '2000-b'],
+          appendOnlyFrom: ['2000-b'],
+        }),
+      ).toEqual([expect.stringContaining('not append-only')]);
+    });
+
+    it('should fail when entries were removed', () => {
+      expect(
+        verifyOrderContent({ actual: ['1000-a'], expected: ['1000-a'], appendOnlyFrom: ['1000-a', '2000-b'] }),
+      ).toEqual([expect.stringContaining('fewer entries')]);
+    });
+
+    it('should not report append-only violations when the content is already inconsistent', () => {
+      expect(
+        verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a'], appendOnlyFrom: ['2000-b'] }),
+      ).toEqual([expect.stringContaining('does not exist')]);
+    });
+  });
+});

packages/sql-tools/src/migration-order.ts

@@ -0,0 +1,104 @@
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join, parse } from 'node:path';
+
+// no extension means kysely ignores this
+export const ORDER_FILENAME = 'ORDER';
+
+const isEqual = (a: string[], b: string[]) => a.length === b.length && a.every((item, i) => b[i] === item);
+
+const findDuplicates = (names: string[]): string[] => [
+  ...new Set(names.filter((name, index) => names.indexOf(name) !== index)),
+];
+
+export const computeOrder = (fileNames: string[]): string[] =>
+  fileNames
+    .filter((name) => name !== ORDER_FILENAME)
+    .map((name) => parse(name).name)
+    .toSorted();
+
+export const parseOrder = (content: string): string[] =>
+  content
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+const findPrefixViolation = (base: string[], current: string[]): string | undefined => {
+  if (base.length > current.length) {
+    return `${ORDER_FILENAME} has fewer entries (${current.length}) than the baseline (${base.length}); migrations cannot be removed`;
+  }
+
+  const index = base.findIndex((name, i) => current[i] !== name);
+  return index === -1
+    ? undefined
+    : `${ORDER_FILENAME} is not append-only: expected "${base[index]}" at position ${index + 1}, found "${current[index]}". New migrations must sort after all existing migrations`;
+};
+
+export type VerifyOrderInput = { actual: string[]; expected: string[]; appendOnlyFrom?: string[] };
+
+export const verifyOrderContent = ({ actual, expected, appendOnlyFrom }: VerifyOrderInput): string[] => {
+  const expectedSet = new Set(expected);
+  const actualSet = new Set(actual);
+
+  const errors = [
+    ...findDuplicates(expected).map((name) => `Duplicate migration name "${name}"`),
+    ...findDuplicates(actual).map((name) => `Duplicate ${ORDER_FILENAME} entry "${name}"`),
+    ...[...expectedSet.difference(actualSet)].map((name) => `Migration "${name}" is missing from ${ORDER_FILENAME}`),
+    ...[...actualSet.difference(expectedSet)].map(
+      (name) => `"${name}" is listed in ${ORDER_FILENAME} but does not exist`,
+    ),
+  ];
+
+  if (errors.length === 0 && !isEqual(actual, expected)) {
+    errors.push(`${ORDER_FILENAME} entries are out of order (expected sorted migration names)`);
+  }
+
+  if (errors.length === 0 && appendOnlyFrom) {
+    const violation = findPrefixViolation(appendOnlyFrom, actual);
+    if (violation) {
+      errors.push(violation);
+    }
+  }
+
+  return errors;
+};
+
+export const listMigrationNames = (folder: string): string[] =>
+  computeOrder(
+    readdirSync(folder, { withFileTypes: true })
+      .filter((entry) => entry.isFile())
+      .map((entry) => entry.name),
+  );
+
+export const readOrder = (folder: string): string[] | undefined => {
+  const path = join(folder, ORDER_FILENAME);
+  return existsSync(path) ? parseOrder(readFileSync(path, 'utf8')) : undefined;
+};
+
+export const writeOrder = (folder: string, names: string[]): void => {
+  writeFileSync(join(folder, ORDER_FILENAME), names.map((name) => `${name}\n`).join(''));
+};
+
+export const syncOrder = (folder: string): { previous?: string[]; next: string[]; changed: boolean } => {
+  const previous = readOrder(folder);
+  const next = listMigrationNames(folder);
+  const changed = previous === undefined || !isEqual(previous, next);
+  if (changed) {
+    writeOrder(folder, next);
+  }
+
+  return { previous, next, changed };
+};
+
+export const maybeSyncOrder = (folder: string): boolean =>
+  existsSync(join(folder, ORDER_FILENAME)) ? syncOrder(folder).changed : false;
+
+export type VerifyOrderOptions = { appendOnlyFrom?: string[] };
+
+export const verifyOrder = (folder: string, { appendOnlyFrom }: VerifyOrderOptions = {}): string[] => {
+  const actual = readOrder(folder);
+  if (actual === undefined) {
+    return [`Missing ${ORDER_FILENAME} file in ${folder} (run \`sql-tools migrations sync-order\` to create it)`];
+  }
+
+  return verifyOrderContent({ actual, expected: listMigrationNames(folder), appendOnlyFrom });
+};

packages/sql-tools/src/migration.ts

@@ -12,6 +12,7 @@ import {
   schemaFromCode,
   schemaFromDatabase,
 } from 'src';
+import { ORDER_FILENAME, maybeSyncOrder } from 'src/migration-order';
 
 type MigrationProps = {
   up: string[];
@@ -168,6 +169,9 @@ export class Migrator {
     mkdirSync(folder, { recursive: true });
     writeFileSync(fullPath, this.#asMigration({ up, down }));
     console.log(`Wrote ${fullPath}`);
+    if (maybeSyncOrder(folder)) {
+      console.log(`Updated ${join(folder, ORDER_FILENAME)}`);
+    }
   }
 
   async #compare() {
@@ -240,6 +244,10 @@ ${downSql}
         console.log(`Removed ${filePath}`);
       }
     }
+
+    if (maybeSyncOrder(sourceFolder)) {
+      console.log(`Updated ${join(sourceFolder, ORDER_FILENAME)}`);
+    }
   }
 
   destroy() {

packages/sql-tools/tsconfig.json

@@ -9,6 +9,7 @@
     "sourceMap": true,
     "strict": true,
     "target": "es2024",
+    "lib": ["es2025"],
     "module": "es2022",
     "moduleResolution": "bundler",
     "declaration": true,