mirror of
https://github.com/immich-app/static-pages.git
synced 2026-09-30 13:23:05 +08:00
refactor: import post (#579)
This commit is contained in:
Jason Rasmussen
@@ -23,11 +23,25 @@ jobs:
|
||||
persist-credentials: false
|
||||
ref: 'main'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
||||
- name: Setup Mise
|
||||
uses: immich-app/devtools/actions/use-mise@7b8610a904d57da241e4ddba17fa62b62b15aed4 # use-mise-action-v2.0.2
|
||||
with:
|
||||
github_token: ${{ steps.generate-token.outputs.token }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
|
||||
- name: Get pnpm store directory
|
||||
id: pnpm-store
|
||||
run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ${{ steps.pnpm-store.outputs.path }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pnpm-store-
|
||||
|
||||
- name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Import post
|
||||
id: import
|
||||
@@ -39,7 +53,7 @@ jobs:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.STATIC_BUCKET_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.STATIC_BUCKET_KEY_SECRET }}
|
||||
R2_PUBLIC_URL: 'https://static.immich.cloud'
|
||||
run: uv run --script import-post.py "$OUTLINE_POST_URL"
|
||||
run: mise import-post "$OUTLINE_POST_URL"
|
||||
|
||||
- name: Create PR
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
|
||||
@@ -65,6 +65,12 @@ depends = [ { task = "watch" }, { task = "open", args = ["{{usage.app}}"] } ]
|
||||
[tasks.watch]
|
||||
run = "pnpm --filter @immich/ui --filter @immich/svelte-markdown-preprocess --parallel run watch"
|
||||
|
||||
[tasks.import-post]
|
||||
dir = "{{config_root}}"
|
||||
run = "pnpm --filter @immich/import-post run import"
|
||||
env = { _.file = ".env" }
|
||||
|
||||
|
||||
[tasks.open]
|
||||
usage = '''
|
||||
arg "<app>" help="Target app"
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
# outline sync stuff
|
||||
OUTLINE_API_KEY=
|
||||
R2_BUCKET_NAME=static
|
||||
R2_ENDPOINT_URL=
|
||||
R2_ACCESS_KEY_ID=
|
||||
R2_SECRET_ACCESS_KEY=
|
||||
R2_PUBLIC_URL=https://static.immich.cloud
|
||||
@@ -7,7 +7,6 @@
|
||||
"start": "vite dev",
|
||||
"build": "npm run sync && vite build && node ../../bin/postbuild.js",
|
||||
"preview": "vite preview",
|
||||
"import-post": "dotenv -- uv run ../../import-post.py",
|
||||
"check": "svelte-check --tsconfig ./tsconfig.json",
|
||||
"uuid": "npx -y uuid v7",
|
||||
"sync": "svelte-kit sync",
|
||||
@@ -23,7 +22,6 @@
|
||||
"@types/node": "^24.13.0",
|
||||
"@vitest/browser": "^4.1.0",
|
||||
"dotenv": "^17.2.3",
|
||||
"dotenv-cli": "^11.0.0",
|
||||
"globals": "^17.0.0",
|
||||
"luxon": "^3.7.2",
|
||||
"marked": "^18.0.0",
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
IMMICH_APP=buy.immich.app
|
||||
PUBLIC_IMMICH_PAY_HOST=https://futopay-test.azurewebsites.net
|
||||
# PUBLIC_IMMICH_SPEC_URL=http://localhost:3000/api/spec.json
|
||||
|
||||
# outline sync stuff
|
||||
OUTLINE_API_KEY=
|
||||
R2_BUCKET_NAME=static
|
||||
R2_ENDPOINT_URL=
|
||||
R2_ACCESS_KEY_ID=
|
||||
R2_SECRET_ACCESS_KEY=
|
||||
R2_PUBLIC_URL=https://static.immich.cloud
|
||||
|
||||
-313
@@ -1,313 +0,0 @@
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "requests == 2.32.5",
|
||||
# "python-frontmatter == 1.1.0",
|
||||
# "mistletoe == 1.5.0",
|
||||
# "pillow == 12.0.0",
|
||||
# "boto3 == 1.40.64",
|
||||
# "pyyaml == 6.0.3"
|
||||
# ]
|
||||
# ///
|
||||
import datetime
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import yaml
|
||||
import time
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from urllib.parse import parse_qs, urljoin, urlparse, urlunparse
|
||||
|
||||
import boto3
|
||||
import frontmatter
|
||||
import mistletoe
|
||||
import requests
|
||||
from mistletoe import span_token
|
||||
from mistletoe.markdown_renderer import MarkdownRenderer, Fragment
|
||||
from PIL import Image
|
||||
|
||||
WEBP_QUALITY = 85
|
||||
BLOG_PREFIX = "blog"
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
OUTPUT_BASE = SCRIPT_DIR / "apps/root.immich.app/src/routes/blog"
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
slug = text.lower().strip()
|
||||
slug = re.sub(r'[^\w\s-]', '', slug)
|
||||
slug = re.sub(r'[-\s]+', '-', slug)
|
||||
return slug.strip('-')
|
||||
|
||||
|
||||
def hash_content(data: bytes) -> str:
|
||||
return hashlib.md5(data).hexdigest()
|
||||
|
||||
|
||||
def extract_id(url: str) -> str:
|
||||
# https://outline.immich.cloud/api/attachments.redirect?id=1315cf2a-80af-4f16-8caa-8bfcf56da74f -> 1315cf2a-80af-4f16-8caa-8bfcf56da74f
|
||||
return parse_qs(urlparse(url).query).get('id', [url])[0]
|
||||
|
||||
|
||||
def download_media(url: str, auth_header: str) -> tuple[bytes, str]:
|
||||
response = requests.get(url, headers={"Authorization": auth_header})
|
||||
response.raise_for_status()
|
||||
content_type = response.headers['Content-Type'] # ensure content type is present
|
||||
return (response.content, content_type)
|
||||
|
||||
|
||||
def convert_to_webp(image_data: bytes, quality: int = WEBP_QUALITY) -> bytes:
|
||||
img = Image.open(BytesIO(image_data))
|
||||
|
||||
if img.mode in ('RGBA', 'LA', 'P'):
|
||||
background = Image.new('RGB', img.size, (255, 255, 255))
|
||||
if img.mode == 'P':
|
||||
img = img.convert('RGBA')
|
||||
background.paste(img, mask=img.split()[-1] if img.mode in ('RGBA', 'LA') else None)
|
||||
img = background
|
||||
elif img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
|
||||
output = BytesIO()
|
||||
img.save(output, format='WEBP', quality=quality)
|
||||
return output.getvalue()
|
||||
|
||||
class FlowList(list):
|
||||
pass
|
||||
|
||||
def represent_flow_list(dumper, data):
|
||||
return dumper.represent_sequence(
|
||||
"tag:yaml.org,2002:seq",
|
||||
data,
|
||||
flow_style=True,
|
||||
)
|
||||
|
||||
yaml.add_representer(FlowList, represent_flow_list, Dumper=yaml.SafeDumper)
|
||||
|
||||
|
||||
class ImageProcessingRenderer(MarkdownRenderer):
|
||||
cover_image: span_token.Image | None = None
|
||||
|
||||
def __init__(self, importer: 'PostImporter', uuid: str, first_as_cover: bool = False):
|
||||
super().__init__()
|
||||
self.importer = importer
|
||||
self.uuid = uuid
|
||||
self.first_as_cover = first_as_cover
|
||||
|
||||
def render_link(self, token: span_token.Link) -> Iterable[Fragment]:
|
||||
original_url = token.target
|
||||
|
||||
# assume all attachment links are videos?
|
||||
if not original_url.startswith('/api/attachments.redirect'):
|
||||
return super().render_link(token)
|
||||
|
||||
attachment_id = extract_id(original_url)
|
||||
|
||||
original_url = urlunparse(self.importer.outline_url_parts._replace(path=token.target))
|
||||
|
||||
video_data, content_type = download_media(original_url, self.importer.outline_auth_header)
|
||||
if content_type != 'video/mp4':
|
||||
print(f"Warning: Expected video/mp4 but got {content_type} for {attachment_id}")
|
||||
return super().render_link(token)
|
||||
|
||||
content_hash = hash_content(video_data)
|
||||
|
||||
print(f"Processing attachment: {attachment_id} ({content_type})")
|
||||
|
||||
s3_key = f"{BLOG_PREFIX}/{self.uuid}/{content_hash}.mp4"
|
||||
self.importer.upload_to_s3(video_data, s3_key, content_type='video/mp4')
|
||||
|
||||
src = f"{self.importer.r2_public_url}/{s3_key}"
|
||||
print(f"Replaced with: {src}")
|
||||
|
||||
title = getattr(token, "title", None)
|
||||
|
||||
# outline puts dimensions in the title
|
||||
if getattr(token, "title", None) and token.title.startswith(" ="):
|
||||
title = None
|
||||
|
||||
template = '<video autoplay src="{src}"{title} controls>Your browser does not support the video tag.</video>'
|
||||
title_template = ' title="{title}"'
|
||||
|
||||
return [Fragment(text=template.format(src=src, title=title_template.format(title=title) if title else ''))]
|
||||
|
||||
|
||||
def render_image(self, token: span_token.Image) -> Iterable[Fragment]:
|
||||
original_url = token.src
|
||||
if not original_url.startswith(('http://', 'https://')):
|
||||
parts = self.importer.outline_url_parts
|
||||
original_url = urlunparse(parts._replace(path=original_url))
|
||||
|
||||
image_data, content_type = download_media(original_url, self.importer.outline_auth_header)
|
||||
|
||||
print(f"Processing attachment: {extract_id(original_url)} ({content_type})")
|
||||
|
||||
content_hash = hash_content(image_data)
|
||||
webp_data = convert_to_webp(image_data)
|
||||
|
||||
s3_key = f"{BLOG_PREFIX}/{self.uuid}/{content_hash}.webp"
|
||||
self.importer.upload_to_s3(webp_data, s3_key, content_type='image/webp')
|
||||
|
||||
token.src = f"{self.importer.r2_public_url}/{s3_key}"
|
||||
|
||||
# outline puts dimensions in the title
|
||||
if getattr(token, "title", None) and token.title.startswith(" ="):
|
||||
token.title = None
|
||||
|
||||
if self.first_as_cover and self.cover_image is None:
|
||||
self.cover_image = token
|
||||
return ""
|
||||
|
||||
return super().render_image(token)
|
||||
|
||||
|
||||
class PostImporter:
|
||||
def __init__(self, post_url: str):
|
||||
self.outline_auth_header = f"Bearer {os.environ["OUTLINE_API_KEY"]}"
|
||||
self.outline_url_parts = urlparse(post_url)
|
||||
self.r2_bucket_name = os.environ["R2_BUCKET_NAME"]
|
||||
self.r2_public_url = os.environ["R2_PUBLIC_URL"].rstrip('/')
|
||||
|
||||
self.s3 = boto3.client(
|
||||
's3',
|
||||
endpoint_url=os.environ["R2_ENDPOINT_URL"],
|
||||
aws_access_key_id=os.environ["R2_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"]
|
||||
)
|
||||
|
||||
self.existing_keys: set[str] = set()
|
||||
self.referenced_keys: set[str] = set()
|
||||
|
||||
def fetch_post_from_outline(self) -> tuple[dict, frontmatter.Post]:
|
||||
post_id = self.outline_url_parts.path.removeprefix('/doc/').rstrip('/')
|
||||
api_url = urlunparse(self.outline_url_parts._replace(path='/api/documents.info'))
|
||||
|
||||
headers = {"Authorization": self.outline_auth_header}
|
||||
response = requests.post(api_url, headers=headers, json={"id": post_id})
|
||||
response.raise_for_status()
|
||||
post = response.json()
|
||||
|
||||
text = post['data']['text'].replace('---', '---', 2)
|
||||
|
||||
post_data = frontmatter.loads(text)
|
||||
|
||||
return post, post_data
|
||||
|
||||
def list_s3_keys(self, prefix: str) -> set[str]:
|
||||
paginator = self.s3.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=self.r2_bucket_name, Prefix=prefix)
|
||||
|
||||
keys: set[str] = set()
|
||||
for page in pages:
|
||||
for obj in page.get('Contents', []):
|
||||
keys.add(obj['Key'])
|
||||
|
||||
return keys
|
||||
|
||||
def delete_unreferenced(self) -> None:
|
||||
stale_keys = self.existing_keys - self.referenced_keys
|
||||
if not stale_keys:
|
||||
return
|
||||
|
||||
self.s3.delete_objects(
|
||||
Bucket=self.r2_bucket_name,
|
||||
Delete={'Objects': [{'Key': key} for key in stale_keys]}
|
||||
)
|
||||
|
||||
def upload_to_s3(self, data: bytes, key: str, content_type: str) -> None:
|
||||
self.referenced_keys.add(key)
|
||||
|
||||
if key in self.existing_keys:
|
||||
return
|
||||
|
||||
response = self.s3.put_object(
|
||||
Bucket=self.r2_bucket_name,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type
|
||||
)
|
||||
etag = response.get('ETag', '').strip('"')
|
||||
print(f"Uploaded: {key}")
|
||||
|
||||
def write_output(self, post_data: frontmatter.Post, folder: str) -> Path:
|
||||
output_dir = OUTPUT_BASE / folder
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_file = output_dir / "+page.md"
|
||||
|
||||
output_file.write_text(frontmatter.dumps(post_data, Dumper=yaml.SafeDumper))
|
||||
|
||||
print(f"\nprettier --write")
|
||||
result = subprocess.run(
|
||||
["pnpm", "exec", "prettier", "--write", str(output_file)],
|
||||
cwd=SCRIPT_DIR,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for line in (result.stdout + result.stderr).splitlines():
|
||||
print(f"> {line}")
|
||||
|
||||
return output_file
|
||||
|
||||
def run(self) -> None:
|
||||
post, post_data = self.fetch_post_from_outline()
|
||||
uuid = post['data']['id']
|
||||
title = post_data.get('title') or post['data']['title']
|
||||
slug = post_data.get('slug') or slugify(title)
|
||||
post_type = post_data.get('type') or 'post'
|
||||
folder = f"({post_type}s)/{slug}"
|
||||
bucket = f"{BLOG_PREFIX}/{uuid}/"
|
||||
output_relative = f"src/routes/blog/{folder}/+page.md"
|
||||
|
||||
print(f"Importing:\n ID: {uuid}\n Title: {title}\n Path: {output_relative}\n Bucket: {bucket}")
|
||||
print("Continue? [Y/n] (continuing in 5s) ", end="", flush=True)
|
||||
ready, _, _ = select.select([sys.stdin], [], [], 5)
|
||||
answer = sys.stdin.readline().strip().lower() if ready else ""
|
||||
print()
|
||||
if answer.startswith("n"):
|
||||
return
|
||||
|
||||
|
||||
self.existing_keys = self.list_s3_keys(f"{bucket}")
|
||||
|
||||
with ImageProcessingRenderer(self, uuid, first_as_cover=post_type != 'release') as renderer:
|
||||
doc = mistletoe.Document(post_data.content)
|
||||
|
||||
post_data.content = renderer.render(doc)
|
||||
post_data['id'] = uuid
|
||||
post_data['title'] = title
|
||||
|
||||
if 'publishedAt' not in post_data:
|
||||
post_data['publishedAt'] = datetime.date.today()
|
||||
post_data['authors'] = FlowList(['Immich Team'])
|
||||
post_data['slug'] = slug
|
||||
|
||||
if renderer.cover_image:
|
||||
post_data['coverUrl'] = renderer.cover_image.src
|
||||
post_data['coverAlt'] = "".join(child.content for child in getattr(renderer.cover_image, "children", []) if hasattr(child, "content"))
|
||||
|
||||
uploaded = len(self.referenced_keys - self.existing_keys)
|
||||
unchanged = len(self.referenced_keys & self.existing_keys)
|
||||
deleted = len(self.existing_keys - self.referenced_keys)
|
||||
print(f"\nBucket stats (uploaded={uploaded}, unchanged={unchanged}, deleted={deleted})")
|
||||
|
||||
self.delete_unreferenced()
|
||||
|
||||
output_file = self.write_output(post_data, folder)
|
||||
|
||||
print(f"\nhttp://localhost:5173/blog/{slug}")
|
||||
|
||||
if output := os.environ.get('GITHUB_OUTPUT'):
|
||||
with open(output, 'a') as o:
|
||||
o.write(f"slug={slug}\n")
|
||||
o.write(f"uuid={uuid}\n")
|
||||
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <outline-post-url>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
PostImporter(sys.argv[1]).run()
|
||||
@@ -0,0 +1,29 @@
|
||||
# @immich/import-post
|
||||
|
||||
Imports an [Outline](https://www.getoutline.com/) document into the blog. It:
|
||||
|
||||
1. Fetches the document from the Outline API.
|
||||
2. Downloads every referenced image/video attachment.
|
||||
3. Optimizes and uploads them to and R2 bucket using md5 hash as filename
|
||||
4. Updates the markdown link to use static.immich.cloud
|
||||
5. Writes the updated markdown to disk
|
||||
6. Runs `prettier --write` on the file
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
pnpm --filter @immich/import-post run import "<outline-post-url>"
|
||||
```
|
||||
|
||||
Required environment variables:
|
||||
|
||||
| Variable | Description |
|
||||
| ---------------------- | ----------------------------------- |
|
||||
| `OUTLINE_API_KEY` | Outline API token |
|
||||
| `R2_BUCKET_NAME` | Target bucket |
|
||||
| `R2_ENDPOINT_URL` | S3-compatible endpoint |
|
||||
| `R2_ACCESS_KEY_ID` | Access key |
|
||||
| `R2_SECRET_ACCESS_KEY` | Secret key |
|
||||
| `R2_PUBLIC_URL` | Public base URL for uploaded assets |
|
||||
|
||||
The post URL may also be supplied via `OUTLINE_POST_URL` instead of an argument.
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "@immich/import-post",
|
||||
"description": "Import Outline documents into the blog, optimizing media for CDN/web delivery",
|
||||
"version": "0.1.0",
|
||||
"license": "MIT",
|
||||
"private": true,
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/immich-app/static-pages.git",
|
||||
"directory": "packages/import-post"
|
||||
},
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"import-post": "./src/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"import": "tsx src/index.ts",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"check": "tsc -p tsconfig.json --noEmit",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.738.0",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"remark-stringify": "^11.0.0",
|
||||
"sharp": "^0.34.2",
|
||||
"unified": "^11.0.5",
|
||||
"unist-util-visit": "^5.0.0",
|
||||
"yaml": "^2.7.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/mdast": "^4.0.4",
|
||||
"@types/node": "^25.9.1",
|
||||
"tsx": "^4.20.0",
|
||||
"typescript": "^6.0.0",
|
||||
"vitest": "^4.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export const WEBP_QUALITY = 85;
|
||||
|
||||
// Outline links to attachments (videos) via this redirect path.
|
||||
export const ATTACHMENT_PREFIX = '/api/attachments.redirect';
|
||||
|
||||
// Matches a date-only string (e.g. `2026-03-02`) so it can be emitted unquoted.
|
||||
export const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/;
|
||||
@@ -0,0 +1,37 @@
|
||||
import { ConfigRepository } from './repositories/config.repository.js';
|
||||
import { MediaRepository } from './repositories/media.repository.js';
|
||||
import { OutlineRepository } from './repositories/outline.repository.js';
|
||||
import { R2Repository } from './repositories/r2.repository.js';
|
||||
import { SystemRepository } from './repositories/system.repository.js';
|
||||
import { ImportService } from './services/import.service.js';
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const postUrl = process.argv[2] ?? process.env.OUTLINE_POST_URL;
|
||||
if (!postUrl) {
|
||||
console.error('Usage: import-post <outline-post-url>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const configRepository = ConfigRepository.create();
|
||||
const config = configRepository.get();
|
||||
|
||||
const systemRepository = new SystemRepository();
|
||||
const outlineRepository = new OutlineRepository(config.outlineApiKey);
|
||||
const mediaRepository = new MediaRepository();
|
||||
const r2Repository = new R2Repository(config.r2);
|
||||
|
||||
const service = new ImportService(
|
||||
configRepository,
|
||||
outlineRepository,
|
||||
systemRepository,
|
||||
mediaRepository,
|
||||
r2Repository,
|
||||
);
|
||||
|
||||
await service.run(postUrl);
|
||||
};
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { R2Config } from '../types.js';
|
||||
|
||||
const required = (name: string): string => {
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${name}`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
type Config = {
|
||||
outlineApiKey: string;
|
||||
rootPath: string;
|
||||
r2: R2Config;
|
||||
};
|
||||
|
||||
export class ConfigRepository {
|
||||
private static instance?: ConfigRepository;
|
||||
|
||||
private config: Config;
|
||||
|
||||
private constructor() {
|
||||
this.config = {
|
||||
outlineApiKey: required('OUTLINE_API_KEY'),
|
||||
rootPath: fileURLToPath(new URL('../../../../', import.meta.url)),
|
||||
r2: {
|
||||
bucket: required('R2_BUCKET_NAME'),
|
||||
endpoint: required('R2_ENDPOINT_URL'),
|
||||
accessKeyId: required('R2_ACCESS_KEY_ID'),
|
||||
secretAccessKey: required('R2_SECRET_ACCESS_KEY'),
|
||||
publicUrl: required('R2_PUBLIC_URL').replace(/\/+$/, ''),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
get() {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
static create() {
|
||||
if (!ConfigRepository.instance) {
|
||||
ConfigRepository.instance = new ConfigRepository();
|
||||
}
|
||||
|
||||
return ConfigRepository.instance;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import sharp from 'sharp';
|
||||
import { WEBP_QUALITY } from '../constants.js';
|
||||
import { OptimizeResult } from '../types.js';
|
||||
|
||||
export class MediaRepository {
|
||||
async optimizeImage(buffer: Buffer): Promise<OptimizeResult> {
|
||||
const output = await sharp(buffer).flatten({ background: '#ffffff' }).webp({ quality: WEBP_QUALITY }).toBuffer();
|
||||
return {
|
||||
buffer: output,
|
||||
extension: 'webp',
|
||||
contentType: 'image/webp',
|
||||
};
|
||||
}
|
||||
|
||||
async optimizeVideo(buffer: Buffer): Promise<OptimizeResult> {
|
||||
// todo
|
||||
const output = buffer;
|
||||
return {
|
||||
buffer: output,
|
||||
extension: 'mp4',
|
||||
contentType: 'video/mp4',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { OutlineDocument } from '../types.js';
|
||||
|
||||
export class OutlineRepository {
|
||||
private readonly authHeader: string;
|
||||
|
||||
constructor(apiKey: string) {
|
||||
this.authHeader = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
async getDocument(postUrl: string): Promise<OutlineDocument> {
|
||||
const parts = new URL(postUrl);
|
||||
const postId = parts.pathname.replace(/^\/doc\//, '').replace(/\/+$/, '');
|
||||
const apiUrl = new URL('/api/documents.info', parts).toString();
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: this.authHeader, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ id: postId }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Outline API error (${response.status} ${response.statusText}) for ${apiUrl}`);
|
||||
}
|
||||
|
||||
const json = (await response.json()) as { data: OutlineDocument };
|
||||
return json.data;
|
||||
}
|
||||
|
||||
async download(url: string) {
|
||||
const response = await fetch(url, { headers: { Authorization: this.authHeader } });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download ${url} (${response.status} ${response.statusText})`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type');
|
||||
if (!contentType) {
|
||||
throw new Error(`Missing Content-Type for ${url}`);
|
||||
}
|
||||
|
||||
return { buffer: Buffer.from(await response.arrayBuffer()), contentType };
|
||||
}
|
||||
|
||||
// https://outline.immich.cloud/api/attachments.redirect?id=1315cf2a-... -> 1315cf2a-...
|
||||
getAttachmentId(url: string): string {
|
||||
try {
|
||||
return new URL(url, 'http://outline.invalid').searchParams.get('id') ?? url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { DeleteObjectsCommand, paginateListObjectsV2, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import type { R2Config } from '../types.js';
|
||||
|
||||
export class R2Repository {
|
||||
private readonly client: S3Client;
|
||||
private readonly bucket: string;
|
||||
|
||||
constructor(config: R2Config) {
|
||||
this.bucket = config.bucket;
|
||||
this.client = new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: config.endpoint,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async listKeys(prefix: string): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
for await (const page of paginateListObjectsV2({ client: this.client }, { Bucket: this.bucket, Prefix: prefix })) {
|
||||
for (const object of page.Contents ?? []) {
|
||||
if (object.Key) {
|
||||
keys.push(object.Key);
|
||||
}
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async upload(key: string, body: Buffer, contentType: string): Promise<void> {
|
||||
await this.client.send(
|
||||
new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: body, ContentType: contentType }),
|
||||
);
|
||||
}
|
||||
|
||||
async deleteKeys(keys: string[]): Promise<void> {
|
||||
if (keys.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: this.bucket,
|
||||
Delete: { Objects: keys.map((Key) => ({ Key })) },
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
export class SystemRepository {
|
||||
md5(buffer: Buffer): string {
|
||||
return createHash('md5').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
write(path: string, content: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, content);
|
||||
}
|
||||
|
||||
append(path: string, content: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
appendFileSync(path, content);
|
||||
}
|
||||
|
||||
format(repoRoot: string, file: string): void {
|
||||
try {
|
||||
const output = execFileSync('pnpm', ['exec', 'prettier', '--write', file], {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
for (const line of output.split('\n')) {
|
||||
if (line.length > 0) {
|
||||
console.log(`> ${line}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const stdout = (error as { stdout?: string }).stdout ?? '';
|
||||
const stderr = (error as { stderr?: string }).stderr ?? '';
|
||||
for (const line of (stdout + stderr).split('\n')) {
|
||||
if (line.length > 0) {
|
||||
console.log(`> ${line}`);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async confirm(options?: { prompt?: string; timeoutSeconds: number }): Promise<boolean> {
|
||||
if (!process.stdin.isTTY) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const { prompt, timeoutSeconds = 5 } = options ?? {};
|
||||
|
||||
if (prompt) {
|
||||
process.stdout.write(prompt + '\n');
|
||||
}
|
||||
|
||||
process.stdout.write(`Continue? [Y/n] (continuing in ${timeoutSeconds}s) `);
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
process.stdin.off('data', onData);
|
||||
process.stdin.pause();
|
||||
};
|
||||
|
||||
const onData = (chunk: Buffer) => {
|
||||
cleanup();
|
||||
resolve(!chunk.toString().trim().toLowerCase().startsWith('n'));
|
||||
};
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
process.stdout.write('\n');
|
||||
resolve(true);
|
||||
}, timeoutSeconds * 1000);
|
||||
|
||||
process.stdin.resume();
|
||||
process.stdin.once('data', onData);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
import { type Mocked, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { ConfigRepository } from '../repositories/config.repository.js';
|
||||
import type { MediaRepository } from '../repositories/media.repository.js';
|
||||
import type { OutlineRepository } from '../repositories/outline.repository.js';
|
||||
import type { R2Repository } from '../repositories/r2.repository.js';
|
||||
import type { SystemRepository } from '../repositories/system.repository.js';
|
||||
import { ImportService } from './import.service.js';
|
||||
|
||||
const POST_URL = 'https://outline.immich/doc/abc';
|
||||
|
||||
type Mock<T extends object> = Mocked<Pick<T, keyof T>>;
|
||||
type Real<T> = T extends Mock<infer U> ? U : never;
|
||||
|
||||
const config = {
|
||||
outlineApiKey: 'key',
|
||||
rootPath: '/tmp/repo',
|
||||
r2: {
|
||||
bucket: 'bucket',
|
||||
endpoint: '',
|
||||
accessKeyId: '',
|
||||
secretAccessKey: '',
|
||||
publicUrl: 'https://static.immich.cloud',
|
||||
},
|
||||
};
|
||||
|
||||
describe(ImportService.name, () => {
|
||||
let configMock: Mock<ConfigRepository>;
|
||||
let outlineMock: Mock<OutlineRepository>;
|
||||
let systemMock: Mock<SystemRepository>;
|
||||
let mediaMock: Mock<MediaRepository>;
|
||||
let r2Mock: Mock<R2Repository>;
|
||||
let sut: ImportService;
|
||||
|
||||
beforeEach(() => {
|
||||
configMock = {
|
||||
get: vi.fn().mockReturnValue(config),
|
||||
};
|
||||
|
||||
outlineMock = {
|
||||
getDocument: vi.fn(),
|
||||
download: vi.fn().mockResolvedValue({ buffer: Buffer.from('image'), contentType: 'image/png' }),
|
||||
getAttachmentId: vi.fn().mockReturnValue('attachment-id'),
|
||||
};
|
||||
|
||||
systemMock = {
|
||||
confirm: vi.fn().mockResolvedValue(true),
|
||||
md5: vi.fn(),
|
||||
write: vi.fn(),
|
||||
format: vi.fn(),
|
||||
append: vi.fn(),
|
||||
};
|
||||
|
||||
mediaMock = {
|
||||
optimizeImage: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ buffer: Buffer.from('webp'), extension: 'webp', contentType: 'image/webp' }),
|
||||
optimizeVideo: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ buffer: Buffer.from('mp4'), extension: 'mp4', contentType: 'video/mp4' }),
|
||||
};
|
||||
|
||||
r2Mock = {
|
||||
listKeys: vi.fn().mockResolvedValue([]),
|
||||
upload: vi.fn(),
|
||||
deleteKeys: vi.fn(),
|
||||
};
|
||||
|
||||
sut = new ImportService(
|
||||
configMock as Real<ConfigRepository>,
|
||||
outlineMock as Real<OutlineRepository>,
|
||||
systemMock as Real<SystemRepository>,
|
||||
mediaMock as Real<MediaRepository>,
|
||||
r2Mock as Real<R2Repository>,
|
||||
);
|
||||
});
|
||||
|
||||
describe('attachment sync', () => {
|
||||
it('should upload a new image', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(r2Mock.upload).toHaveBeenCalledTimes(1);
|
||||
expect(r2Mock.upload).toHaveBeenCalledWith('blog/post-id-1/file-hash-1.webp', expect.any(Buffer), 'image/webp');
|
||||
});
|
||||
|
||||
it('should not re-upload an image already present in the bucket', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
r2Mock.listKeys.mockResolvedValue(['blog/post-id-1/file-hash-1.webp']);
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(r2Mock.upload).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should upload duplicate images within a document only once', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
---
|
||||
|
||||
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(r2Mock.upload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should delete orphaned attachments that are no longer referenced', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
r2Mock.listKeys.mockResolvedValue(['blog/post-id-1/file-hash-1.webp', 'blog/post-id-1/orphan.webp']);
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(r2Mock.deleteKeys).toHaveBeenCalledWith(['blog/post-id-1/orphan.webp']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('metadata', () => {
|
||||
it('should default the published date to today', async () => {
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining(`publishedAt: ${today}`),
|
||||
);
|
||||
});
|
||||
|
||||
it('should use the published date from front matter', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
publishedAt: 2020-01-02
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('publishedAt: 2020-01-02'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should default the title to the document title', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Document Title',
|
||||
text: `---
|
||||
slug: test
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('title: Document Title'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should use the title from front matter', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Document Title',
|
||||
text: `---
|
||||
title: Front Matter Title
|
||||
slug: test
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('title: Front Matter Title'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should derive the slug from the title by default', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Hello World',
|
||||
text: `---
|
||||
title: Hello World
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.stringContaining('(posts)/hello-world/+page.md'),
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
|
||||
it('should use the slug from front matter', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Hello World',
|
||||
text: `---
|
||||
title: Hello World
|
||||
slug: custom-slug
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.stringContaining('(posts)/custom-slug/+page.md'),
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('post type', () => {
|
||||
it.each([
|
||||
['release', 'releases'],
|
||||
['post', 'posts'],
|
||||
['announcement', 'announcements'],
|
||||
])('should write a "%s" post to the (%s) folder', async (type, folder) => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
type: ${type}
|
||||
---
|
||||
|
||||
Body.`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.stringContaining(`(${folder})/test/+page.md`),
|
||||
expect.any(String),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cover image', () => {
|
||||
it('should use the first image as the cover for non-release posts', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
type: post
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('coverUrl: https://static.immich.cloud/blog/post-id-1/file-hash-1.webp'),
|
||||
);
|
||||
expect(systemMock.write).toHaveBeenCalledWith(expect.any(String), expect.stringContaining('coverAlt: Cover alt'));
|
||||
});
|
||||
|
||||
it('should remove the cover image from the body', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
type: post
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(expect.any(String), expect.not.stringContaining('
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(expect.any(String), expect.stringContaining('coverAlt: First'));
|
||||
});
|
||||
|
||||
it('should not set a cover for release posts', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
type: release
|
||||
---
|
||||
|
||||
`,
|
||||
});
|
||||
systemMock.md5.mockReturnValue('file-hash-1');
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(expect.any(String), expect.not.stringContaining('coverUrl'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('markdown', () => {
|
||||
it('should not escape angle-bracket placeholders with a backslash', async () => {
|
||||
outlineMock.getDocument.mockResolvedValue({
|
||||
id: 'post-id-1',
|
||||
title: 'Test',
|
||||
text: `---
|
||||
title: Test
|
||||
slug: test
|
||||
type: release
|
||||
---
|
||||
|
||||
<example placeholder?>`,
|
||||
});
|
||||
|
||||
await sut.run(POST_URL);
|
||||
|
||||
expect(systemMock.write).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.stringContaining('<example placeholder?>'),
|
||||
);
|
||||
expect(systemMock.write).toHaveBeenCalledWith(expect.any(String), expect.not.stringContaining(String.raw`\<`));
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
import type { Html, Image, Link, Parent } from 'mdast';
|
||||
import { join } from 'node:path';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
import remarkParse from 'remark-parse';
|
||||
import remarkStringify from 'remark-stringify';
|
||||
import { unified } from 'unified';
|
||||
import { visit } from 'unist-util-visit';
|
||||
import { Document, isSeq, parse, Scalar, visit as visitYaml } from 'yaml';
|
||||
import { ATTACHMENT_PREFIX, DATE_ONLY } from '../constants.js';
|
||||
import type { ConfigRepository } from '../repositories/config.repository.js';
|
||||
import type { MediaRepository } from '../repositories/media.repository.js';
|
||||
import type { OutlineRepository } from '../repositories/outline.repository.js';
|
||||
import type { R2Repository } from '../repositories/r2.repository.js';
|
||||
import type { SystemRepository } from '../repositories/system.repository.js';
|
||||
import type { MarkdownDocument, OutlineAttachment, ParsedDocument } from '../types.js';
|
||||
|
||||
type Cover = {
|
||||
url: string;
|
||||
alt: string;
|
||||
};
|
||||
|
||||
const cleanTitle = (title: string | null | undefined): string | undefined =>
|
||||
title && !title.startsWith(' =') ? title : undefined;
|
||||
|
||||
const slugify = (text: string): string =>
|
||||
text
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replaceAll(/[^\w\s-]/g, '')
|
||||
.replaceAll(/[-\s]+/g, '-')
|
||||
.replaceAll(/^-+|-+$/g, '');
|
||||
|
||||
const parseYml = (text: string): ParsedDocument => {
|
||||
const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/.exec(text);
|
||||
if (!match) {
|
||||
return { data: {}, content: text };
|
||||
}
|
||||
const data = (parse(match[1]) as Record<string, unknown> | null) ?? {};
|
||||
return { data, content: match[2] };
|
||||
};
|
||||
|
||||
const serializeYml = (data: Record<string, unknown>, content: string): string => {
|
||||
const doc = new Document(data, { sortMapEntries: true });
|
||||
|
||||
// Keep `authors: [Immich Team]` on a single flow-style line.
|
||||
const authors = doc.get('authors', true);
|
||||
if (isSeq(authors)) {
|
||||
authors.flow = true;
|
||||
}
|
||||
|
||||
// Emit date-only values unquoted (e.g. `publishedAt: 2026-03-02`).
|
||||
visitYaml(doc, {
|
||||
Scalar(_, node) {
|
||||
if (typeof node.value === 'string' && DATE_ONLY.test(node.value)) {
|
||||
node.type = Scalar.PLAIN;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const body = content.replace(/^\s+/, '').replace(/\s+$/, '');
|
||||
return `---\n${doc.toString()}---\n\n${body}\n`;
|
||||
};
|
||||
|
||||
const markdown = unified()
|
||||
.use(remarkParse)
|
||||
.use(remarkGfm)
|
||||
.use(remarkStringify, { bullet: '-', emphasis: '_', strong: '*', rule: '-' });
|
||||
|
||||
export class ImportService {
|
||||
constructor(
|
||||
private readonly configRepository: ConfigRepository,
|
||||
private readonly outlineRepository: OutlineRepository,
|
||||
private readonly systemRepository: SystemRepository,
|
||||
private readonly mediaRepository: MediaRepository,
|
||||
private readonly r2Repository: R2Repository,
|
||||
) {}
|
||||
|
||||
async run(postUrl: string): Promise<void> {
|
||||
const { rootPath: repoRoot, r2 } = this.configRepository.get();
|
||||
|
||||
const post = await this.outlineRepository.getDocument(postUrl);
|
||||
const { data, content } = parseYml(post.text);
|
||||
const uuid = post.id;
|
||||
const title = data.title || post.title;
|
||||
const postType = data.type || 'post';
|
||||
const slug = data.slug || slugify(title);
|
||||
const folder = `(${postType}s)/${slug}`;
|
||||
const bucketFolder = `blog/${uuid}`;
|
||||
const outputRelative = `src/routes/blog/${folder}/+page.md`;
|
||||
|
||||
console.log(`Importing:
|
||||
ID: ${uuid}
|
||||
Title: ${title}
|
||||
Path: ${outputRelative}
|
||||
Bucket: ${bucketFolder}
|
||||
`);
|
||||
|
||||
if (!(await this.systemRepository.confirm())) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existingKeys = new Set(await this.r2Repository.listKeys(bucketFolder));
|
||||
const referencedKeys = new Set<string>();
|
||||
|
||||
const document = markdown.parse(content);
|
||||
const firstAsCover = postType !== 'release';
|
||||
|
||||
let cover: Cover | undefined;
|
||||
for (const attachment of this.getAttachments(document)) {
|
||||
const url = new URL(attachment.url, postUrl).toString();
|
||||
const { buffer, contentType: originalContentType } = await this.outlineRepository.download(url);
|
||||
const attachmentId = this.outlineRepository.getAttachmentId(url);
|
||||
|
||||
if (attachment.type === 'video' && originalContentType !== 'video/mp4') {
|
||||
console.warn(`Warning: Expected video/mp4 but got ${originalContentType} for ${attachmentId}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`Processing attachment: ${attachmentId} (${originalContentType})`);
|
||||
|
||||
const hash = this.systemRepository.md5(buffer);
|
||||
|
||||
const {
|
||||
buffer: body,
|
||||
extension,
|
||||
contentType,
|
||||
} = await (attachment.type === 'video'
|
||||
? this.mediaRepository.optimizeVideo(buffer)
|
||||
: this.mediaRepository.optimizeImage(buffer));
|
||||
|
||||
const filename = `${hash}.${extension}`;
|
||||
|
||||
const key = `${bucketFolder}/${filename}`;
|
||||
if (!existingKeys.has(key) && !referencedKeys.has(key)) {
|
||||
await this.r2Repository.upload(key, body, contentType);
|
||||
console.log(`Uploaded: ${key}`);
|
||||
}
|
||||
referencedKeys.add(key);
|
||||
const newUrl = `${r2.publicUrl}/${key}`;
|
||||
|
||||
if (firstAsCover && !cover) {
|
||||
cover = { url: newUrl, alt: attachment.alt };
|
||||
attachment.remove();
|
||||
} else {
|
||||
attachment.update(newUrl, cleanTitle(attachment.title));
|
||||
}
|
||||
}
|
||||
|
||||
// remark escapes `<` in text as `\<`; keep it bare to match the source
|
||||
const rendered = markdown.stringify(document).replaceAll(String.raw`\<`, '<');
|
||||
|
||||
const metadata: Record<string, unknown> = { ...data, id: uuid, title, slug, authors: ['Immich Team'] };
|
||||
if (metadata.publishedAt instanceof Date) {
|
||||
metadata.publishedAt = metadata.publishedAt.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
if (!metadata.publishedAt) {
|
||||
metadata.publishedAt = new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
if (cover) {
|
||||
metadata.coverUrl = cover.url;
|
||||
metadata.coverAlt = cover.alt;
|
||||
}
|
||||
|
||||
const staleKeys = [...existingKeys].filter((key) => !referencedKeys.has(key));
|
||||
const uploadedKeys = [...referencedKeys].filter((key) => !existingKeys.has(key));
|
||||
const unchanged = referencedKeys.size - uploadedKeys.length;
|
||||
console.log(
|
||||
`\nBucket stats (uploaded=${uploadedKeys.length}, unchanged=${unchanged}, deleted=${staleKeys.length})`,
|
||||
);
|
||||
await this.r2Repository.deleteKeys(staleKeys);
|
||||
|
||||
const outputFile = join(repoRoot, 'apps/root.immich.app/src/routes/blog', folder, '+page.md');
|
||||
this.systemRepository.write(outputFile, serializeYml(metadata, rendered));
|
||||
this.systemRepository.format(repoRoot, outputFile);
|
||||
|
||||
console.log(`\nhttp://localhost:5173/blog/${slug}`);
|
||||
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
this.systemRepository.append(process.env.GITHUB_OUTPUT, `slug=${slug}\nuuid=${uuid}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
getAttachments(document: MarkdownDocument): OutlineAttachment[] {
|
||||
const attachments: OutlineAttachment[] = [];
|
||||
|
||||
visit(document, 'image', (node, index, parent) => {
|
||||
if (!parent || typeof index !== 'number') {
|
||||
return;
|
||||
}
|
||||
|
||||
attachments.push({
|
||||
url: node.url,
|
||||
type: 'image',
|
||||
alt: node.alt ?? '',
|
||||
title: node.title,
|
||||
update: (url, title) => {
|
||||
node.title = title;
|
||||
node.url = url;
|
||||
},
|
||||
remove: () => this.removeNode(document, parent, node),
|
||||
});
|
||||
});
|
||||
|
||||
visit(document, 'link', (node, index, parent) => {
|
||||
if (!parent || typeof index !== 'number' || !node.url.startsWith(ATTACHMENT_PREFIX)) {
|
||||
return;
|
||||
}
|
||||
|
||||
attachments.push({
|
||||
url: node.url,
|
||||
type: 'video',
|
||||
alt: '',
|
||||
title: node.title,
|
||||
update: (src, title) => {
|
||||
const titleAttr = title ? ` title="${title}"` : '';
|
||||
const html: Html = {
|
||||
type: 'html',
|
||||
value: `<video autoplay src="${src}"${titleAttr} controls>Your browser does not support the video tag.</video>`,
|
||||
};
|
||||
const at = parent.children.indexOf(node);
|
||||
if (at !== -1) {
|
||||
parent.children[at] = html;
|
||||
}
|
||||
},
|
||||
remove: () => this.removeNode(document, parent, node),
|
||||
});
|
||||
});
|
||||
|
||||
return attachments;
|
||||
}
|
||||
|
||||
private removeNode(document: MarkdownDocument, parent: Parent, node: Image | Link): void {
|
||||
const index = parent.children.indexOf(node);
|
||||
if (index !== -1) {
|
||||
parent.children.splice(index, 1);
|
||||
}
|
||||
// Drop the wrapping paragraph if removing the node left it empty.
|
||||
if (parent.type === 'paragraph' && parent.children.length === 0) {
|
||||
visit(document, 'paragraph', (candidate, candidateIndex, candidateParent) => {
|
||||
if (candidate === parent && candidateParent && typeof candidateIndex === 'number') {
|
||||
candidateParent.children.splice(candidateIndex, 1);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { Root } from 'mdast';
|
||||
|
||||
export type R2Config = {
|
||||
bucket: string;
|
||||
endpoint: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
publicUrl: string;
|
||||
};
|
||||
|
||||
export type OutlineDocument = {
|
||||
id: string;
|
||||
title: string;
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type ParsedDocument = {
|
||||
data: {
|
||||
title?: string;
|
||||
description?: string;
|
||||
publishedAt?: string;
|
||||
type?: string;
|
||||
slug?: string;
|
||||
};
|
||||
content: string;
|
||||
};
|
||||
|
||||
export type OptimizeResult = { buffer: Buffer; extension: string; contentType: string };
|
||||
|
||||
export type MarkdownDocument = Root;
|
||||
|
||||
export type AttachmentType = 'image' | 'video';
|
||||
|
||||
export type OutlineAttachment = {
|
||||
url: string;
|
||||
type: AttachmentType;
|
||||
alt: string;
|
||||
title: string | null | undefined;
|
||||
update(url: string, title: string | null | undefined): void;
|
||||
remove(): void;
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"declaration": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"sourceMap": false,
|
||||
"strict": true,
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ES2023",
|
||||
"outDir": "./dist",
|
||||
"types": ["node"],
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["dist", "**/*.spec.ts", "vitest.config.ts"]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ['{src,test}/**/*.{test,spec}.{js,ts}'],
|
||||
},
|
||||
});
|
||||
Generated
+1542
-181
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user