diff --git a/.github/workflows/import-post.yml b/.github/workflows/import-post.yml index 1ebcb155..c47900ed 100644 --- a/.github/workflows/import-post.yml +++ b/.github/workflows/import-post.yml @@ -39,10 +39,7 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.STATIC_BUCKET_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.STATIC_BUCKET_KEY_SECRET }} R2_PUBLIC_URL: 'https://static.immich.cloud' - run: uv run --script import-post.py - - - name: Format - run: pnpm --filter root.immich.app run format:fix + run: uv run --script import-post.py "$OUTLINE_POST_URL" - name: Create PR uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 diff --git a/apps/root.immich.app/example.env b/apps/root.immich.app/example.env index e9dc242b..a4813d48 100644 --- a/apps/root.immich.app/example.env +++ b/apps/root.immich.app/example.env @@ -1,5 +1,4 @@ # outline sync stuff -OUTLINE_POST_URL=https://outline.immich.cloud/doc/march-recap-XXXXXXXXXX OUTLINE_API_KEY= R2_BUCKET_NAME=static R2_ENDPOINT_URL= diff --git a/import-post.py b/import-post.py index 5a322cc2..c5212d9b 100644 --- a/import-post.py +++ b/import-post.py @@ -13,13 +13,14 @@ import hashlib import os import re import select +import subprocess import sys import yaml import time from io import BytesIO from pathlib import Path from typing import Iterable -from urllib.parse import urljoin, urlparse, urlunparse +from urllib.parse import parse_qs, urljoin, urlparse, urlunparse import boto3 import frontmatter @@ -46,6 +47,11 @@ def hash_content(data: bytes) -> str: return hashlib.md5(data).hexdigest() +def extract_id(url: str) -> str: + # https://outline.immich.cloud/api/attachments.redirect?id=1315cf2a-80af-4f16-8caa-8bfcf56da74f -> 1315cf2a-80af-4f16-8caa-8bfcf56da74f + return parse_qs(urlparse(url).query).get('id', [url])[0] + + def download_media(url: str, auth_header: str) -> tuple[bytes, str]: response = requests.get(url, headers={"Authorization": auth_header}) response.raise_for_status() @@ -98,16 +104,18 @@ class ImageProcessingRenderer(MarkdownRenderer): if not original_url.startswith('/api/attachments.redirect'): return super().render_link(token) + attachment_id = extract_id(original_url) + original_url = urlunparse(self.importer.outline_url_parts._replace(path=token.target)) video_data, content_type = download_media(original_url, self.importer.outline_auth_header) if content_type != 'video/mp4': - print(f"Warning: Expected video/mp4 but got {content_type} for {original_url}") + print(f"Warning: Expected video/mp4 but got {content_type} for {attachment_id}") return super().render_link(token) content_hash = hash_content(video_data) - print(f"Processing video: {original_url} ({content_type})") + print(f"Processing attachment: {attachment_id} ({content_type})") s3_key = f"{BLOG_PREFIX}/{self.uuid}/{content_hash}.mp4" self.importer.upload_to_s3(video_data, s3_key, content_type='video/mp4') @@ -135,7 +143,7 @@ class ImageProcessingRenderer(MarkdownRenderer): image_data, content_type = download_media(original_url, self.importer.outline_auth_header) - print(f"Processing image: {original_url} ({content_type})") + print(f"Processing attachment: {extract_id(original_url)} ({content_type})") content_hash = hash_content(image_data) webp_data = convert_to_webp(image_data) @@ -145,8 +153,6 @@ class ImageProcessingRenderer(MarkdownRenderer): token.src = f"{self.importer.r2_public_url}/{s3_key}" - print(f"Replaced with: {token.src}") - # outline puts dimensions in the title if getattr(token, "title", None) and token.title.startswith(" ="): token.title = None @@ -159,9 +165,9 @@ class ImageProcessingRenderer(MarkdownRenderer): class PostImporter: - def __init__(self): + def __init__(self, post_url: str): self.outline_auth_header = f"Bearer {os.environ["OUTLINE_API_KEY"]}" - self.outline_url_parts = urlparse(os.environ["OUTLINE_POST_URL"]) + self.outline_url_parts = urlparse(post_url) self.r2_bucket_name = os.environ["R2_BUCKET_NAME"] self.r2_public_url = os.environ["R2_PUBLIC_URL"].rstrip('/') @@ -172,6 +178,9 @@ class PostImporter: aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"] ) + self.existing_keys: set[str] = set() + self.referenced_keys: set[str] = set() + def fetch_post_from_outline(self) -> tuple[dict, frontmatter.Post]: post_id = self.outline_url_parts.path.removeprefix('/doc/').rstrip('/') api_url = urlunparse(self.outline_url_parts._replace(path='/api/documents.info')) @@ -187,25 +196,33 @@ class PostImporter: return post, post_data - def clear_s3_directory(self, prefix: str) -> None: - print(f"Clearing S3 directory: {prefix}") - + def list_s3_keys(self, prefix: str) -> set[str]: paginator = self.s3.get_paginator('list_objects_v2') pages = paginator.paginate(Bucket=self.r2_bucket_name, Prefix=prefix) - objects_to_delete = [] + keys: set[str] = set() for page in pages: - if 'Contents' in page: - objects_to_delete.extend([{'Key': obj['Key']} for obj in page['Contents']]) + for obj in page.get('Contents', []): + keys.add(obj['Key']) - if objects_to_delete: - self.s3.delete_objects( - Bucket=self.r2_bucket_name, - Delete={'Objects': objects_to_delete} - ) - print(f"Deleted {len(objects_to_delete)} objects from {prefix}") + return keys + + def delete_unreferenced(self) -> None: + stale_keys = self.existing_keys - self.referenced_keys + if not stale_keys: + return + + self.s3.delete_objects( + Bucket=self.r2_bucket_name, + Delete={'Objects': [{'Key': key} for key in stale_keys]} + ) def upload_to_s3(self, data: bytes, key: str, content_type: str) -> None: + self.referenced_keys.add(key) + + if key in self.existing_keys: + return + response = self.s3.put_object( Bucket=self.r2_bucket_name, Key=key, @@ -213,7 +230,7 @@ class PostImporter: ContentType=content_type ) etag = response.get('ETag', '').strip('"') - print(f"Uploaded to S3: {key} (ETag: {etag})") + print(f"Uploaded: {key}") def write_output(self, post_data: frontmatter.Post, folder: str) -> Path: output_dir = OUTPUT_BASE / folder @@ -221,6 +238,18 @@ class PostImporter: output_file = output_dir / "+page.md" output_file.write_text(frontmatter.dumps(post_data, Dumper=yaml.SafeDumper)) + + print(f"\nprettier --write") + result = subprocess.run( + ["pnpm", "exec", "prettier", "--write", str(output_file)], + cwd=SCRIPT_DIR, + check=True, + capture_output=True, + text=True, + ) + for line in (result.stdout + result.stderr).splitlines(): + print(f"> {line}") + return output_file def run(self) -> None: @@ -230,8 +259,10 @@ class PostImporter: slug = post_data.get('slug') or slugify(title) post_type = post_data.get('type') or 'post' folder = f"({post_type}s)/{slug}" + bucket = f"{BLOG_PREFIX}/{uuid}/" + output_relative = f"src/routes/blog/{folder}/+page.md" - print(f"Importing:\n ID: {uuid}\n Title: {title}\n Path: routes/blog/{folder}/+page.md\n") + print(f"Importing:\n ID: {uuid}\n Title: {title}\n Path: {output_relative}\n Bucket: {bucket}") print("Continue? [Y/n] (continuing in 5s) ", end="", flush=True) ready, _, _ = select.select([sys.stdin], [], [], 5) answer = sys.stdin.readline().strip().lower() if ready else "" @@ -239,7 +270,8 @@ class PostImporter: if answer.startswith("n"): return - self.clear_s3_directory(f"{BLOG_PREFIX}/{uuid}/") + + self.existing_keys = self.list_s3_keys(f"{bucket}") with ImageProcessingRenderer(self, uuid, first_as_cover=post_type != 'release') as renderer: doc = mistletoe.Document(post_data.content) @@ -257,10 +289,16 @@ class PostImporter: post_data['coverUrl'] = renderer.cover_image.src post_data['coverAlt'] = "".join(child.content for child in getattr(renderer.cover_image, "children", []) if hasattr(child, "content")) + uploaded = len(self.referenced_keys - self.existing_keys) + unchanged = len(self.referenced_keys & self.existing_keys) + deleted = len(self.existing_keys - self.referenced_keys) + print(f"\nBucket stats (uploaded={uploaded}, unchanged={unchanged}, deleted={deleted})") + + self.delete_unreferenced() + output_file = self.write_output(post_data, folder) - print(f"\nPost written to: {output_file}") - print(f"Post slug: {slug}") + print(f"\nhttp://localhost:5173/blog/{slug}") if output := os.environ.get('GITHUB_OUTPUT'): with open(output, 'a') as o: @@ -268,4 +306,8 @@ class PostImporter: o.write(f"uuid={uuid}\n") -PostImporter().run() +if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + +PostImporter(sys.argv[1]).run()