feat: virtual docs module (#784)

* feat: virtual docs module

* chore: add readme

packages/svelte-markdown-preprocess/README.md

@@ -0,0 +1,175 @@
+# @immich/svelte-markdown-preprocess
+
+Renders markdown as Svelte components, and serves the collected front matter and headings back to the
+app through a virtual module.
+
+The package has two halves, and they are used together:
+
+| Export                     | Kind                | Configured in      |
+| -------------------------- | ------------------- | ------------------ |
+| `svelteMarkdownPreprocess` | Svelte preprocessor | `svelte.config.js` |
+| `svelteMarkdownVite`       | Vite plugin         | `vite.config.ts`   |
+
+The preprocessor turns each `.md` file into a Svelte component and wraps it in an optional layout. The Vite
+plugin scans the same files and serves them as `virtual:docs`, so a layout is handed the parsed doc
+for its own page and the app can list every doc without importing any markdown into the browser.
+
+## Setup
+
+Four changes are needed to start using them.
+
+### 1. `svelte.config.js`
+
+Register the preprocessor and tell SvelteKit that `.md` files are routes.
+
+```js
+import { svelteMarkdownPreprocess } from '@immich/svelte-markdown-preprocess';
+import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
+
+const config = {
+  extensions: ['.svelte', '.md'],
+  preprocess: [
+    svelteMarkdownPreprocess({
+      layouts: {
+        default: '$lib/components/MarkdownPage.svelte',
+      },
+    }),
+    vitePreprocess(),
+  ],
+};
+
+export default config;
+```
+
+`layouts.default` is used for every markdown file. A file can pick another one with a `layout` key in
+its front matter, matching a key in `layouts`.
+
+### 2. `vite.config.ts`
+
+```ts
+import { svelteMarkdownVite } from '@immich/svelte-markdown-preprocess';
+import { sveltekit } from '@sveltejs/kit/vite';
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+  plugins: [sveltekit(), svelteMarkdownVite()],
+});
+```
+
+### 3. `src/app.d.ts`
+
+Pull in the ambient declaration for `virtual:docs`.
+
+```ts
+/// <reference types="@immich/svelte-markdown-preprocess/virtual" />
+```
+
+### 4. The layout component
+
+The preprocessor looks up the current page and passes it as a `doc` prop.
+
+```svelte
+<script lang="ts">
+  import type { ClientDoc } from '@immich/svelte-markdown-preprocess';
+  import type { Snippet } from 'svelte';
+
+  type Props = {
+    doc?: ClientDoc;
+    children?: Snippet;
+  };
+
+  const { doc, children }: Props = $props();
+</script>
+
+<h1>{doc?.attributes.title}</h1>
+
+{@render children?.()}
+```
+
+`doc` is optional because markdown outside of `src/routes` still gets a layout, but has no entry in
+the collection.
+
+## The virtual module
+
+```ts
+import { getDoc, getDocs } from 'virtual:docs';
+
+const all = getDocs();
+const one = getDoc('(shell)/blog/(posts)/sync-v2/+page.md');
+```
+
+`getDoc` takes the doc's `path` - the file's location relative to `src/routes`, layout groups
+included. Both are generic over the doc type, so pass yours to get it back:
+
+```ts
+const posts = getDocs<BlogPost>();
+```
+
+Every doc is at least a `ClientDoc`:
+
+```ts
+type ClientDoc = {
+  path: string;
+  attributes: FrontMatterAttributes; // parsed front matter
+  headers: DocHeader[]; // level 2 and 3 headings, ids matching the rendered anchors
+};
+```
+
+Markdown bodies are never serialized into the module, so no post content reaches the browser.
+
+## Validating and reshaping docs
+
+`onDoc` runs at build time for each file and decides what ships. It receives a `ServerDoc`, which is a
+`ClientDoc` plus the markdown `body`. Throwing fails the build, and returning `undefined` leaves the
+doc out of the collection.
+
+```ts
+svelteMarkdownVite({
+  onDoc: ({ path, attributes, headers }) => {
+    const parsed = FrontMatterSchema.safeParse(attributes);
+    if (!parsed.success) {
+      throw new Error(`${path} has invalid front matter`);
+    }
+
+    return { path, attributes, headers, ...parsed.data };
+  },
+});
+```
+
+A doc may gain properties, but never lose the ones on `ClientDoc` - the return type is constrained to
+`T extends ClientDoc`, and `getDoc` keys the collection on `path`.
+
+## Options
+
+`svelteMarkdownPreprocess(options)`
+
+| Option       | Default           | Purpose                                        |
+| ------------ | ----------------- | ---------------------------------------------- |
+| `extensions` | `['.md', '.mdx']` | file extensions to treat as markdown           |
+| `layouts`    | `{}`              | layout component per front matter `layout` key |
+
+`svelteMarkdownVite(options)`
+
+| Option       | Default           | Purpose                                     |
+| ------------ | ----------------- | ------------------------------------------- |
+| `extensions` | `['.md', '.mdx']` | file extensions to collect                  |
+| `onDoc`      | strips `body`     | validate and reshape a doc, or leave it out |
+
+The module id (`virtual:docs`) and the scanned directory (`src/routes`) are fixed, and exported as
+`VIRTUAL_ID` and `DOCS_DIR`.
+
+## Utilities
+
+| Export                       | Purpose                                                       |
+| ---------------------------- | ------------------------------------------------------------- |
+| `getHeaders(body, levels?)`  | headings of a markdown body, with anchor ids                  |
+| `getHrefFromPath(path)`      | route of a page, with layout groups removed                   |
+| `getIdFromText(text)`        | anchor id used for a heading, so links match what is rendered |
+| `isMarkdownPath(path, ext?)` | whether a path is markdown                                    |
+| `parseFrontMatter(content)`  | `{ attributes, body }` of a markdown file                     |
+| `markedText(markdown)`       | markdown rendered to plain text, for search indexes           |
+
+## Development
+
+Editing a markdown file invalidates `virtual:docs` and triggers a full reload, so headings and front
+matter stay current without restarting the dev server.

packages/svelte-markdown-preprocess/package.json

@@ -15,6 +15,7 @@
   },
   "files": [
     "dist",
+    "virtual.d.ts",
     "!dist/**/*.test.*",
     "!dist/**/*.spec.*"
   ],
@@ -24,6 +25,9 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./virtual": {
+      "types": "./virtual.d.ts"
     }
   },
   "peerDependencies": {

packages/svelte-markdown-preprocess/src/index.spec.ts

@@ -56,17 +56,45 @@ describe(svelteMarkdownPreprocess.name, () => {
       });
     });
 
-    it('should pass attributes to the layout component', async () => {
+    it('should look up the doc by path and pass it to the layout component', async () => {
       const result = await svelteMarkdownPreprocess({
         layouts: { default: '$lib/layouts/DefaultLayout.svelte' },
       }).markup({
-        filename: 'test.md',
+        filename: '/app/src/routes/(shell)/blog/(posts)/sync-v2/+page.md',
         content: `---\ntitle: Test\n---\n# Hello`,
       });
 
-      expect(result).toMatchObject({ code: expect.stringContaining(`const attributes = {"title":"Test"}`) });
+      expect(result).toMatchObject({ code: expect.stringContaining(`import { getDoc } from 'virtual:docs'`) });
+      expect(result).toMatchObject({
+        code: expect.stringContaining(`const doc = getDoc('(shell)/blog/(posts)/sync-v2/+page.md')`),
+      });
       // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation
-      expect(result).toMatchObject({ code: expect.stringContaining(`<Layout {attributes}>`) });
+      expect(result).toMatchObject({ code: expect.stringContaining(`<Layout {doc}>`) });
+    });
+
+    it('should look up a doc for markdown that is not a route page', async () => {
+      const result = await svelteMarkdownPreprocess({
+        layouts: { default: '$lib/layouts/DefaultLayout.svelte' },
+      }).markup({
+        filename: '/app/src/routes/components/markdown/Example.md',
+        content: `# Hello`,
+      });
+
+      expect(result).toMatchObject({
+        code: expect.stringContaining(`const doc = getDoc('components/markdown/Example.md')`),
+      });
+    });
+
+    it('should not look up a doc for markdown outside of the configured dir', async () => {
+      const result = await svelteMarkdownPreprocess({
+        layouts: { default: '$lib/layouts/DefaultLayout.svelte' },
+      }).markup({
+        filename: '/app/other/Example.md',
+        content: `# Hello`,
+      });
+
+      expect(result).toMatchObject({ code: expect.not.stringContaining('getDoc') });
+      expect(result).toMatchObject({ code: expect.stringContaining(`<Layout>`) });
     });
   });
 

packages/svelte-markdown-preprocess/src/index.ts

@@ -1,3 +1,4 @@
 export * from './markdown.js';
+export * from './svelte-preprocess.js';
 export * from './utility.js';
-export * from './vite-preprocess.js';
+export * from './vite.js';

packages/svelte-markdown-preprocess/src/svelte-preprocess.ts

@@ -1,8 +1,9 @@
 /* eslint-disable unicorn/prefer-await */
-import fm from 'front-matter';
 import { Marked } from 'marked';
 import type { PreprocessorGroup } from 'svelte/compiler';
 import { markedSvelte } from './markdown.js';
+import { DOCS_DIR, VIRTUAL_ID } from './vite.js';
+import { parseFrontMatter, type FrontMatterAttributes } from './utility.js';
 
 type MaybePromise<T> = Promise<T> | T;
 
@@ -13,14 +14,9 @@ export type FileWithFrontMatter = { filename: string; attributes: FrontMatterAtt
 export type FileWithScriptBody = FileWithFrontMatter & { scriptBody: string };
 export type FileWithMarkup = FileWithScriptBody & { markup: string };
 export type FileWithImages = FileWithMarkup & { images: MarkdownImage[] };
-export type FileWithLayout = FileWithImages & { layout?: string };
+export type FileWithLayout = FileWithImages & { layout?: string; path?: string };
 export type FileWithSvelte = FileWithLayout & { svelte: string };
 
-export type FrontMatterAttributes = {
-  layout?: string;
-  [key: string]: unknown;
-};
-
 export type SvelteMarkdownPreprocessLayouts = {
   _?: string;
   default?: string;
@@ -80,9 +76,7 @@ export class SvelteMarkdownPreprocess {
   }
 
   parseFrontMatter({ filename, content }: FileWithContent): MaybePromise<FileWithFrontMatter> {
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    // @ts-expect-error
-    const { attributes, body } = fm(content) as { attributes: FrontMatterAttributes; body: string };
+    const { attributes, body } = parseFrontMatter(content);
     return { filename, body, attributes };
   }
 
@@ -131,7 +125,17 @@ export class SvelteMarkdownPreprocess {
   parseLayout(file: FileWithImages): MaybePromise<FileWithLayout> {
     const layoutKey = file.attributes.layout;
     const layout = layoutKey ? this.#layouts[layoutKey] : (this.#layouts.default ?? this.#layouts._);
-    return { ...file, layout };
+    return { ...file, layout, path: this.parsePath(file.filename) };
+  }
+
+  parsePath(filename: string): string | undefined {
+    const path = filename.replaceAll('\\', '/');
+    const index = path.lastIndexOf(`/${DOCS_DIR}/`);
+    if (index === -1) {
+      return;
+    }
+
+    return path.slice(index + DOCS_DIR.length + 2);
   }
 
   parseSvelte(file: FileWithLayout): MaybePromise<FileWithSvelte> {
@@ -145,12 +149,17 @@ export class SvelteMarkdownPreprocess {
     return [
       `  import { Markdown } from '@immich/ui';`,
       file.layout ? `  import Layout from '${file.layout}';` : undefined,
+      this.hasDoc(file) ? `  import { getDoc } from '${VIRTUAL_ID}';` : undefined,
       ...file.images.map((image) => `  import ${image.name} from '${image.path}';`),
     ];
   }
 
+  hasDoc(file: FileWithLayout): boolean {
+    return !!(file.layout && file.path);
+  }
+
   scriptExtras(file: FileWithLayout): Array<string | undefined> {
-    return file.layout ? [`  const attributes = ${JSON.stringify(file.attributes)};`] : [];
+    return this.hasDoc(file) ? [`  const doc = getDoc('${file.path}');`] : [];
   }
 
   createSvelteScript(file: FileWithLayout): string {
@@ -167,7 +176,8 @@ export class SvelteMarkdownPreprocess {
 
   createSvelteTemplate(file: FileWithLayout): string {
     // eslint-disable-next-line unicorn/no-incorrect-template-string-interpolation
-    return (file.layout ? [`<Layout {attributes}>`, file.markup, '</Layout>'] : [file.markup]).join('\n');
+    const open = this.hasDoc(file) ? `<Layout {doc}>` : `<Layout>`;
+    return (file.layout ? [open, file.markup, '</Layout>'] : [file.markup]).join('\n');
   }
 }
 

packages/svelte-markdown-preprocess/src/utility.ts

@@ -1,3 +1,16 @@
+import fm from 'front-matter';
+
+export type FrontMatterAttributes = {
+  layout?: string;
+  [key: string]: unknown;
+};
+
+export const parseFrontMatter = (content: string) => {
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-expect-error
+  return fm(content) as { attributes: FrontMatterAttributes; body: string };
+};
+
 export const getIdFromText = (text: string) => {
   let id = text
     .toLowerCase()

packages/svelte-markdown-preprocess/src/vite.spec.ts

@@ -0,0 +1,71 @@
+import { describe, expect, test } from 'vitest';
+import { getHeaders, getHrefFromPath, isMarkdownPath } from './vite.js';
+
+describe('getHeaders', () => {
+  test('keeps level two and three headings with anchors matching the rendered ids', () => {
+    expect(getHeaders('# Title\n\n## Using `restic`\n\nText\n\n### Sub heading\n\n#### Deep\n')).toEqual([
+      { id: 'using', text: 'Using restic', level: 2 },
+      { id: 'sub-heading', text: 'Sub heading', level: 3 },
+    ]);
+  });
+
+  test('collects the requested levels only', () => {
+    expect(getHeaders('# One\n\n## Two\n\n### Three\n', [1, 3])).toEqual([
+      { id: 'one', text: 'One', level: 1 },
+      { id: 'three', text: 'Three', level: 3 },
+    ]);
+  });
+
+  test('ignores headings inside fenced code blocks', () => {
+    const body = ['## Config', '', '```bash', '# comment', '## not a heading', '```', '', '## Usage', ''].join('\n');
+
+    expect(getHeaders(body)).toEqual([
+      { id: 'config', text: 'Config', level: 2 },
+      { id: 'usage', text: 'Usage', level: 2 },
+    ]);
+  });
+
+  test('replaces emoji shortcodes in the text, but not in the anchor', () => {
+    expect(getHeaders('## Bye bye interns! :wave:\n')).toEqual([
+      { id: 'bye-bye-interns', text: 'Bye bye interns! \u{1F44B}', level: 2 },
+    ]);
+  });
+
+  test('returns an empty list when there are no headings', () => {
+    expect(getHeaders('Just a paragraph.\n')).toEqual([]);
+  });
+});
+
+describe('getHrefFromPath', () => {
+  test('drops layout groups', () => {
+    expect(getHrefFromPath('(shell)/blog/(posts)/sync-v2/+page.md')).toBe('/blog/sync-v2');
+  });
+
+  test('handles a page without any groups', () => {
+    expect(getHrefFromPath('getting-started/install/+page.md')).toBe('/getting-started/install');
+  });
+
+  test('handles the root page', () => {
+    expect(getHrefFromPath('+page.md')).toBe('/');
+  });
+
+  test('normalizes windows separators', () => {
+    expect(getHrefFromPath(String.raw`(shell)\blog\(posts)\sync-v2\+page.md`)).toBe('/blog/sync-v2');
+  });
+});
+
+describe('isMarkdownPath', () => {
+  test('accepts any markdown file, not just route pages', () => {
+    expect(isMarkdownPath('(docs)/components/markdown/Example.md')).toBe(true);
+    expect(isMarkdownPath('(shell)/blog/(posts)/sync-v2/+page.md')).toBe(true);
+  });
+
+  test('rejects other extensions', () => {
+    expect(isMarkdownPath('routes/+page.svelte')).toBe(false);
+  });
+
+  test('honors custom extensions', () => {
+    expect(isMarkdownPath('notes/a.markdown', ['.markdown'])).toBe(true);
+    expect(isMarkdownPath('notes/a.md', ['.markdown'])).toBe(false);
+  });
+});

packages/svelte-markdown-preprocess/src/vite.ts

@@ -0,0 +1,164 @@
+import { Marked, Parser, type Token, type Tokens } from 'marked';
+import { emojify } from 'node-emoji';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import type { Plugin } from 'vite';
+import { markedSvelte } from './markdown.js';
+import { getIdFromText, parseFrontMatter, type FrontMatterAttributes } from './utility.js';
+
+export type DocHeader = {
+  id: string;
+  text: string;
+  level: number;
+};
+
+export type ClientDoc = {
+  path: string;
+  attributes: FrontMatterAttributes;
+  headers: DocHeader[];
+};
+
+export type ServerDoc = ClientDoc & {
+  body: string;
+};
+
+export type SvelteMarkdownViteOptions<T> = {
+  /**
+  defaults to `['.md', '.mdx']`
+  */
+  extensions?: string[];
+  /**
+  validate and reshape a single doc. return `undefined` to leave it out, and throw to fail the build.
+  a doc may gain properties, but never lose the ones on `ClientDoc`
+  */
+  onDoc?: (doc: ServerDoc) => T | undefined;
+};
+
+export const VIRTUAL_ID = 'virtual:docs';
+
+export const DOCS_DIR = 'src/routes';
+
+const RESOLVED_ID = '\0' + VIRTUAL_ID;
+
+const GROUP = /^\(.*\)$/;
+
+const md = new Marked().use(markedSvelte());
+
+const inlineText = (tokens: Token[]): string =>
+  tokens
+    .map((token) => {
+      if ('tokens' in token && token.tokens) {
+        return inlineText(token.tokens);
+      }
+
+      return 'text' in token ? emojify(token.text) : '';
+    })
+    .join('');
+
+export const getHeaders = (body: string, levels: number[] = [2, 3]): DocHeader[] => {
+  const headers: DocHeader[] = [];
+
+  for (const token of md.lexer(body)) {
+    if (token.type !== 'heading') {
+      continue;
+    }
+
+    const { depth, tokens } = token as Tokens.Heading;
+    if (!levels.includes(depth)) {
+      continue;
+    }
+
+    headers.push({
+      id: getIdFromText(Parser.parseInline(tokens, md.defaults)),
+      text: inlineText(tokens),
+      level: depth,
+    });
+  }
+
+  return headers;
+};
+
+export const getHrefFromPath = (path: string) => {
+  const segments = path
+    .replaceAll('\\', '/')
+    .split('/')
+    .slice(0, -1)
+    .filter((segment) => !GROUP.test(segment));
+
+  return '/' + segments.join('/');
+};
+
+export const isMarkdownPath = (path: string, extensions: string[] = ['.md', '.mdx']) =>
+  extensions.some((extension) => path.endsWith(extension));
+
+const readDocs = (dir: string, extensions: string[]) =>
+  readdirSync(dir, { recursive: true, encoding: 'utf8' })
+    .map((entry) => entry.replaceAll('\\', '/'))
+    .filter((entry) => isMarkdownPath(entry, extensions))
+    .toSorted();
+
+const asDoc = (dir: string, path: string): ServerDoc => {
+  const { attributes, body } = parseFrontMatter(readFileSync(join(dir, path), 'utf8'));
+
+  return { path, attributes, headers: getHeaders(body), body };
+};
+
+const serialize = (value: unknown) =>
+  JSON.stringify(value)
+    .replaceAll('<', String.raw`\u003c`)
+    .replaceAll('\u{2028}', String.raw`\u2028`)
+    .replaceAll('\u{2029}', String.raw`\u2029`);
+
+export const svelteMarkdownVite = <T extends ClientDoc = ClientDoc>(options?: SvelteMarkdownViteOptions<T>): Plugin => {
+  const { extensions = ['.md', '.mdx'], onDoc = ({ body: _body, ...doc }: ServerDoc) => doc as T } = options ?? {};
+
+  let root: string;
+
+  return {
+    name: '@immich/svelte-markdown-preprocess:docs',
+
+    configResolved(config) {
+      root = resolve(config.root, DOCS_DIR);
+    },
+
+    resolveId(id) {
+      if (id === VIRTUAL_ID) {
+        return RESOLVED_ID;
+      }
+    },
+
+    load(id) {
+      if (id !== RESOLVED_ID) {
+        return;
+      }
+
+      const docs = readDocs(root, extensions)
+        .map((path) => onDoc(asDoc(root, path)))
+        .filter((doc) => doc !== undefined);
+
+      return [
+        `const docs = ${serialize(docs)};`,
+        `const pathMap = new Map(docs.map((doc) => [doc.path, doc]));`,
+        `export const getDocs = () => docs;`,
+        `export const getDoc = (ref) => pathMap.get(ref);`,
+      ].join('\n');
+    },
+
+    configureServer(server) {
+      server.watcher.on('all', (_event, file) => {
+        if (!isMarkdownPath(file, extensions)) {
+          return;
+        }
+
+        for (const environment of Object.values(server.environments)) {
+          const module = environment.moduleGraph.getModuleById(RESOLVED_ID);
+          if (module) {
+            environment.moduleGraph.invalidateModule(module);
+          }
+        }
+
+        server.ws.send({ type: 'full-reload' });
+      });
+    },
+  };
+};

packages/svelte-markdown-preprocess/virtual.d.ts

@@ -0,0 +1,6 @@
+declare module 'virtual:docs' {
+  type Doc = import('@immich/svelte-markdown-preprocess').ClientDoc;
+
+  export const getDocs: <T extends Doc = Doc>() => T[];
+  export const getDoc: <T extends Doc = Doc>(ref: string) => T | undefined;
+}