From 6d757b3dfabfaf5662dbacea3418808957dc5379 Mon Sep 17 00:00:00 2001 From: bo0tzz Date: Tue, 11 Aug 2026 14:40:33 +0200 Subject: [PATCH] feat(sql-tools): maintain and verify a migration ORDER file (#688) --- packages/sql-tools/README.md | 17 +++ packages/sql-tools/src/bin/cli.ts | 40 ++++++- packages/sql-tools/src/index.ts | 1 + .../sql-tools/src/migration-order.spec.ts | 87 +++++++++++++++ packages/sql-tools/src/migration-order.ts | 104 ++++++++++++++++++ packages/sql-tools/src/migration.ts | 8 ++ packages/sql-tools/tsconfig.json | 1 + 7 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 packages/sql-tools/src/migration-order.spec.ts create mode 100644 packages/sql-tools/src/migration-order.ts diff --git a/packages/sql-tools/README.md b/packages/sql-tools/README.md index 653c1fbe..8f770d55 100644 --- a/packages/sql-tools/README.md +++ b/packages/sql-tools/README.md @@ -7,3 +7,20 @@ Kysely-based tools and utilities for managing postgres schema. ```bash npm i @immich/sql-tools ``` + +## Migration order file + +If an `ORDER` file exists in the migration folder, `migrations create`, `migrations generate`, and `migrations revert` keep it in sync: one migration name per line, in the (sorted) order the migrator will run them. The file gives concurrent migration PRs a merge conflict instead of silently landing out of order, and gives CI something cheap to verify. Every file in the folder except `ORDER` itself is treated as a migration — stray files will fail verification, deliberately. + +```bash +# create or refresh the file (defaults to src/schema/migrations, override with --source-folder) +sql-tools migrations sync-order + +# verify it matches the migration files on disk +sql-tools migrations verify-order + +# additionally verify no entries were inserted or removed relative to a baseline (e.g. the merge base in CI) +sql-tools migrations verify-order --append-only-from /tmp/base-order +``` + +Repositories that don't have an `ORDER` file are unaffected. diff --git a/packages/sql-tools/src/bin/cli.ts b/packages/sql-tools/src/bin/cli.ts index a1c78a26..31e228f6 100644 --- a/packages/sql-tools/src/bin/cli.ts +++ b/packages/sql-tools/src/bin/cli.ts @@ -1,7 +1,9 @@ #!/usr/bin/env node import { Command } from 'commander'; import { sql } from 'kysely'; -import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { ORDER_FILENAME, parseOrder, readOrder, syncOrder, verifyOrder } from 'src/migration-order'; import { Migrator } from 'src/migration'; const withMigrator = @@ -11,6 +13,9 @@ const withMigrator = async function (...args: any[]) { const command: Command = args.at(-1); const options = command.optsWithGlobals(); + if (!options.url) { + throw new Error(`Missing required option '-u, --url '`); + } const migrator = new Migrator({ connectionParams: { connectionType: 'url', url: options.url }, @@ -24,7 +29,7 @@ const withMigrator = const program = new Command('sql-tools'); program - .requiredOption('-u, --url ', 'Database connection url') + .option('-u, --url ', 'Database connection url') .option( '-f, --folder ', 'Path to the runnable (compiled) migration files', @@ -65,6 +70,37 @@ migrations ) .action(withMigrator((migrator, _, [path]) => migrator.create(join(process.cwd(), path ?? 'src/Migration'), [], []))); +migrations + .command('sync-order') + .description(`Regenerate the ${ORDER_FILENAME} file from the migration files on disk`) + .action((_: unknown, command: Command) => { + const folder = resolve(process.cwd(), command.optsWithGlobals().sourceFolder); + const { changed, next } = syncOrder(folder); + console.log( + changed + ? `Wrote ${join(folder, ORDER_FILENAME)} (${next.length} migrations)` + : `${ORDER_FILENAME} is already up to date (${next.length} migrations)`, + ); + }); + +migrations + .command('verify-order') + .description(`Verify the ${ORDER_FILENAME} file matches the migration files on disk`) + .option('--append-only-from ', `Baseline ${ORDER_FILENAME} file that must be a prefix of the current one`) + .action((options: { appendOnlyFrom?: string }, command: Command) => { + const folder = resolve(process.cwd(), command.optsWithGlobals().sourceFolder); + const appendOnlyFrom = options.appendOnlyFrom + ? parseOrder(readFileSync(options.appendOnlyFrom, 'utf8')) + : undefined; + + const errors = verifyOrder(folder, { appendOnlyFrom }); + if (errors.length > 0) { + throw new Error(errors.map((error) => `- ${error}`).join('\n')); + } + + console.log(`${ORDER_FILENAME} is consistent (${readOrder(folder)?.length ?? 0} migrations)`); + }); + migrations .command('generate') .description('Generate a new migration file that contains the UP and DOWN queries to migrate the schema') diff --git a/packages/sql-tools/src/index.ts b/packages/sql-tools/src/index.ts index 3b781fbf..a0837e68 100644 --- a/packages/sql-tools/src/index.ts +++ b/packages/sql-tools/src/index.ts @@ -24,6 +24,7 @@ export * from 'src/decorators/trigger.decorator'; export * from 'src/decorators/unique.decorator'; export * from 'src/decorators/update-date-column.decorator'; export * from 'src/migration'; +export * from 'src/migration-order'; export * from 'src/naming/default.naming'; export * from 'src/naming/naming.interface'; export * from 'src/register-enum'; diff --git a/packages/sql-tools/src/migration-order.spec.ts b/packages/sql-tools/src/migration-order.spec.ts new file mode 100644 index 00000000..0c7f626a --- /dev/null +++ b/packages/sql-tools/src/migration-order.spec.ts @@ -0,0 +1,87 @@ +import { computeOrder, ORDER_FILENAME, parseOrder, verifyOrderContent } from 'src/migration-order'; +import { describe, expect, it } from 'vitest'; + +describe(ORDER_FILENAME, () => { + describe(computeOrder.name, () => { + it('should list migration names sorted and without extensions', () => { + expect(computeOrder(['2000-b.ts', '1000-a.ts'])).toEqual(['1000-a', '2000-b']); + }); + + it('should ignore the ORDER file', () => { + expect(computeOrder(['1000-a.ts', ORDER_FILENAME])).toEqual(['1000-a']); + }); + + it('should include stray files so they fail verification loudly', () => { + expect(computeOrder(['1000-a.ts', '.DS_Store', 'README.md'])).toEqual(['.DS_Store', '1000-a', 'README']); + }); + + it('should keep colliding names so verification fails loudly', () => { + expect(computeOrder(['1000-a.ts', '1000-a.js'])).toEqual(['1000-a', '1000-a']); + }); + }); + + describe(parseOrder.name, () => { + it('should ignore blank lines and surrounding whitespace', () => { + expect(parseOrder('1000-a\n\n 2000-b \n')).toEqual(['1000-a', '2000-b']); + }); + }); + + describe(verifyOrderContent.name, () => { + it('should pass when the content matches', () => { + expect(verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a', '2000-b'] })).toEqual([]); + }); + + it('should fail on duplicate names', () => { + expect(verifyOrderContent({ actual: ['1000-a', '1000-a'], expected: ['1000-a', '1000-a'] })).toEqual([ + expect.stringContaining('Duplicate migration name "1000-a"'), + expect.stringContaining(`Duplicate ${ORDER_FILENAME} entry "1000-a"`), + ]); + }); + + it('should fail when a migration is missing from the file', () => { + expect(verifyOrderContent({ actual: ['1000-a'], expected: ['1000-a', '2000-b'] })).toEqual([ + expect.stringContaining(`"2000-b" is missing from ${ORDER_FILENAME}`), + ]); + }); + + it('should fail when the file lists a migration that does not exist', () => { + expect(verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a'] })).toEqual([ + expect.stringContaining('does not exist'), + ]); + }); + + it('should fail when the entries are out of order', () => { + expect(verifyOrderContent({ actual: ['2000-b', '1000-a'], expected: ['1000-a', '2000-b'] })).toEqual([ + expect.stringContaining('out of order'), + ]); + }); + + it('should pass when the baseline is a prefix', () => { + const lists = { actual: ['1000-a', '2000-b'], expected: ['1000-a', '2000-b'] }; + expect(verifyOrderContent({ ...lists, appendOnlyFrom: ['1000-a'] })).toEqual([]); + expect(verifyOrderContent({ ...lists, appendOnlyFrom: ['1000-a', '2000-b'] })).toEqual([]); + }); + + it('should fail when an entry was added before existing ones', () => { + expect( + verifyOrderContent({ + actual: ['1500-a', '2000-b'], + expected: ['1500-a', '2000-b'], + appendOnlyFrom: ['2000-b'], + }), + ).toEqual([expect.stringContaining('not append-only')]); + }); + + it('should fail when entries were removed', () => { + expect( + verifyOrderContent({ actual: ['1000-a'], expected: ['1000-a'], appendOnlyFrom: ['1000-a', '2000-b'] }), + ).toEqual([expect.stringContaining('fewer entries')]); + }); + + it('should not report append-only violations when the content is already inconsistent', () => { + expect( + verifyOrderContent({ actual: ['1000-a', '2000-b'], expected: ['1000-a'], appendOnlyFrom: ['2000-b'] }), + ).toEqual([expect.stringContaining('does not exist')]); + }); + }); +}); diff --git a/packages/sql-tools/src/migration-order.ts b/packages/sql-tools/src/migration-order.ts new file mode 100644 index 00000000..70cc86ea --- /dev/null +++ b/packages/sql-tools/src/migration-order.ts @@ -0,0 +1,104 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, parse } from 'node:path'; + +// no extension means kysely ignores this +export const ORDER_FILENAME = 'ORDER'; + +const isEqual = (a: string[], b: string[]) => a.length === b.length && a.every((item, i) => b[i] === item); + +const findDuplicates = (names: string[]): string[] => [ + ...new Set(names.filter((name, index) => names.indexOf(name) !== index)), +]; + +export const computeOrder = (fileNames: string[]): string[] => + fileNames + .filter((name) => name !== ORDER_FILENAME) + .map((name) => parse(name).name) + .toSorted(); + +export const parseOrder = (content: string): string[] => + content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); + +const findPrefixViolation = (base: string[], current: string[]): string | undefined => { + if (base.length > current.length) { + return `${ORDER_FILENAME} has fewer entries (${current.length}) than the baseline (${base.length}); migrations cannot be removed`; + } + + const index = base.findIndex((name, i) => current[i] !== name); + return index === -1 + ? undefined + : `${ORDER_FILENAME} is not append-only: expected "${base[index]}" at position ${index + 1}, found "${current[index]}". New migrations must sort after all existing migrations`; +}; + +export type VerifyOrderInput = { actual: string[]; expected: string[]; appendOnlyFrom?: string[] }; + +export const verifyOrderContent = ({ actual, expected, appendOnlyFrom }: VerifyOrderInput): string[] => { + const expectedSet = new Set(expected); + const actualSet = new Set(actual); + + const errors = [ + ...findDuplicates(expected).map((name) => `Duplicate migration name "${name}"`), + ...findDuplicates(actual).map((name) => `Duplicate ${ORDER_FILENAME} entry "${name}"`), + ...[...expectedSet.difference(actualSet)].map((name) => `Migration "${name}" is missing from ${ORDER_FILENAME}`), + ...[...actualSet.difference(expectedSet)].map( + (name) => `"${name}" is listed in ${ORDER_FILENAME} but does not exist`, + ), + ]; + + if (errors.length === 0 && !isEqual(actual, expected)) { + errors.push(`${ORDER_FILENAME} entries are out of order (expected sorted migration names)`); + } + + if (errors.length === 0 && appendOnlyFrom) { + const violation = findPrefixViolation(appendOnlyFrom, actual); + if (violation) { + errors.push(violation); + } + } + + return errors; +}; + +export const listMigrationNames = (folder: string): string[] => + computeOrder( + readdirSync(folder, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name), + ); + +export const readOrder = (folder: string): string[] | undefined => { + const path = join(folder, ORDER_FILENAME); + return existsSync(path) ? parseOrder(readFileSync(path, 'utf8')) : undefined; +}; + +export const writeOrder = (folder: string, names: string[]): void => { + writeFileSync(join(folder, ORDER_FILENAME), names.map((name) => `${name}\n`).join('')); +}; + +export const syncOrder = (folder: string): { previous?: string[]; next: string[]; changed: boolean } => { + const previous = readOrder(folder); + const next = listMigrationNames(folder); + const changed = previous === undefined || !isEqual(previous, next); + if (changed) { + writeOrder(folder, next); + } + + return { previous, next, changed }; +}; + +export const maybeSyncOrder = (folder: string): boolean => + existsSync(join(folder, ORDER_FILENAME)) ? syncOrder(folder).changed : false; + +export type VerifyOrderOptions = { appendOnlyFrom?: string[] }; + +export const verifyOrder = (folder: string, { appendOnlyFrom }: VerifyOrderOptions = {}): string[] => { + const actual = readOrder(folder); + if (actual === undefined) { + return [`Missing ${ORDER_FILENAME} file in ${folder} (run \`sql-tools migrations sync-order\` to create it)`]; + } + + return verifyOrderContent({ actual, expected: listMigrationNames(folder), appendOnlyFrom }); +}; diff --git a/packages/sql-tools/src/migration.ts b/packages/sql-tools/src/migration.ts index 41c959b5..2048d779 100644 --- a/packages/sql-tools/src/migration.ts +++ b/packages/sql-tools/src/migration.ts @@ -12,6 +12,7 @@ import { schemaFromCode, schemaFromDatabase, } from 'src'; +import { ORDER_FILENAME, maybeSyncOrder } from 'src/migration-order'; type MigrationProps = { up: string[]; @@ -168,6 +169,9 @@ export class Migrator { mkdirSync(folder, { recursive: true }); writeFileSync(fullPath, this.#asMigration({ up, down })); console.log(`Wrote ${fullPath}`); + if (maybeSyncOrder(folder)) { + console.log(`Updated ${join(folder, ORDER_FILENAME)}`); + } } async #compare() { @@ -240,6 +244,10 @@ ${downSql} console.log(`Removed ${filePath}`); } } + + if (maybeSyncOrder(sourceFolder)) { + console.log(`Updated ${join(sourceFolder, ORDER_FILENAME)}`); + } } destroy() { diff --git a/packages/sql-tools/tsconfig.json b/packages/sql-tools/tsconfig.json index a7574499..a5e98dff 100644 --- a/packages/sql-tools/tsconfig.json +++ b/packages/sql-tools/tsconfig.json @@ -9,6 +9,7 @@ "sourceMap": true, "strict": true, "target": "es2024", + "lib": ["es2025"], "module": "es2022", "moduleResolution": "bundler", "declaration": true,