immich-app/static-pages
1
0
Fork 0
mirror of https://github.com/immich-app/static-pages.git synced 2026-09-30 13:23:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: import-post (#578)

Browse Source
This commit is contained in:
Jason Rasmussen
2026-06-12 12:45:25 -04:00
committed by GitHub
parent 345e526305
commit 5b7f44bd3c
3 changed files with 69 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/import-post.yml
+1 -4
View File
@@ -39,10 +39,7 @@ jobs:
R2_ACCESS_KEY_ID: ${{ secrets.STATIC_BUCKET_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.STATIC_BUCKET_KEY_SECRET }}
R2_PUBLIC_URL: 'https://static.immich.cloud'
run: uv run --script import-post.py
- name: Format
run: pnpm --filter root.immich.app run format:fix
run: uv run --script import-post.py "$OUTLINE_POST_URL"
- name: Create PR
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
apps/root.immich.app/example.env
-1
View File
@@ -1,5 +1,4 @@
# outline sync stuff
OUTLINE_POST_URL=https://outline.immich.cloud/doc/march-recap-XXXXXXXXXX
OUTLINE_API_KEY=
R2_BUCKET_NAME=static
R2_ENDPOINT_URL=
import-post.py
+68 -26
View File
@@ -13,13 +13,14 @@ import hashlib
import os
import re
import select
import subprocess
import sys
import yaml
import time
from io import BytesIO
from pathlib import Path
from typing import Iterable
from urllib.parse import urljoin, urlparse, urlunparse
from urllib.parse import parse_qs, urljoin, urlparse, urlunparse
import boto3
import frontmatter
@@ -46,6 +47,11 @@ def hash_content(data: bytes) -> str:
return hashlib.md5(data).hexdigest()
def extract_id(url: str) -> str:
# https://outline.immich.cloud/api/attachments.redirect?id=1315cf2a-80af-4f16-8caa-8bfcf56da74f -> 1315cf2a-80af-4f16-8caa-8bfcf56da74f
return parse_qs(urlparse(url).query).get('id', [url])[0]
def download_media(url: str, auth_header: str) -> tuple[bytes, str]:
response = requests.get(url, headers={"Authorization": auth_header})
response.raise_for_status()
@@ -98,16 +104,18 @@ class ImageProcessingRenderer(MarkdownRenderer):
if not original_url.startswith('/api/attachments.redirect'):
return super().render_link(token)
attachment_id = extract_id(original_url)
original_url = urlunparse(self.importer.outline_url_parts._replace(path=token.target))
video_data, content_type = download_media(original_url, self.importer.outline_auth_header)
if content_type != 'video/mp4':
print(f"Warning: Expected video/mp4 but got {content_type} for {original_url}")
print(f"Warning: Expected video/mp4 but got {content_type} for {attachment_id}")
return super().render_link(token)
content_hash = hash_content(video_data)
print(f"Processing video: {original_url} ({content_type})")
print(f"Processing attachment: {attachment_id} ({content_type})")
s3_key = f"{BLOG_PREFIX}/{self.uuid}/{content_hash}.mp4"
self.importer.upload_to_s3(video_data, s3_key, content_type='video/mp4')
@@ -135,7 +143,7 @@ class ImageProcessingRenderer(MarkdownRenderer):
image_data, content_type = download_media(original_url, self.importer.outline_auth_header)
print(f"Processing image: {original_url} ({content_type})")
print(f"Processing attachment: {extract_id(original_url)} ({content_type})")
content_hash = hash_content(image_data)
webp_data = convert_to_webp(image_data)
@@ -145,8 +153,6 @@ class ImageProcessingRenderer(MarkdownRenderer):
token.src = f"{self.importer.r2_public_url}/{s3_key}"
print(f"Replaced with: {token.src}")
# outline puts dimensions in the title
if getattr(token, "title", None) and token.title.startswith(" ="):
token.title = None
@@ -159,9 +165,9 @@ class ImageProcessingRenderer(MarkdownRenderer):
class PostImporter:
def __init__(self):
def __init__(self, post_url: str):
self.outline_auth_header = f"Bearer {os.environ["OUTLINE_API_KEY"]}"
self.outline_url_parts = urlparse(os.environ["OUTLINE_POST_URL"])
self.outline_url_parts = urlparse(post_url)
self.r2_bucket_name = os.environ["R2_BUCKET_NAME"]
self.r2_public_url = os.environ["R2_PUBLIC_URL"].rstrip('/')
@@ -172,6 +178,9 @@ class PostImporter:
aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"]
)
self.existing_keys: set[str] = set()
self.referenced_keys: set[str] = set()
def fetch_post_from_outline(self) -> tuple[dict, frontmatter.Post]:
post_id = self.outline_url_parts.path.removeprefix('/doc/').rstrip('/')
api_url = urlunparse(self.outline_url_parts._replace(path='/api/documents.info'))
@@ -187,25 +196,33 @@ class PostImporter:
return post, post_data
def clear_s3_directory(self, prefix: str) -> None:
print(f"Clearing S3 directory: {prefix}")
def list_s3_keys(self, prefix: str) -> set[str]:
paginator = self.s3.get_paginator('list_objects_v2')
pages = paginator.paginate(Bucket=self.r2_bucket_name, Prefix=prefix)
objects_to_delete = []
keys: set[str] = set()
for page in pages:
if 'Contents' in page:
objects_to_delete.extend([{'Key': obj['Key']} for obj in page['Contents']])
for obj in page.get('Contents', []):
keys.add(obj['Key'])
if objects_to_delete:
self.s3.delete_objects(
Bucket=self.r2_bucket_name,
Delete={'Objects': objects_to_delete}
)
print(f"Deleted {len(objects_to_delete)} objects from {prefix}")
return keys
def delete_unreferenced(self) -> None:
stale_keys = self.existing_keys - self.referenced_keys
if not stale_keys:
return
self.s3.delete_objects(
Bucket=self.r2_bucket_name,
Delete={'Objects': [{'Key': key} for key in stale_keys]}
)
def upload_to_s3(self, data: bytes, key: str, content_type: str) -> None:
self.referenced_keys.add(key)
if key in self.existing_keys:
return
response = self.s3.put_object(
Bucket=self.r2_bucket_name,
Key=key,
@@ -213,7 +230,7 @@ class PostImporter:
ContentType=content_type
)
etag = response.get('ETag', '').strip('"')
print(f"Uploaded to S3: {key} (ETag: {etag})")
print(f"Uploaded: {key}")
def write_output(self, post_data: frontmatter.Post, folder: str) -> Path:
output_dir = OUTPUT_BASE / folder
@@ -221,6 +238,18 @@ class PostImporter:
output_file = output_dir / "+page.md"
output_file.write_text(frontmatter.dumps(post_data, Dumper=yaml.SafeDumper))
print(f"\nprettier --write")
result = subprocess.run(
["pnpm", "exec", "prettier", "--write", str(output_file)],
cwd=SCRIPT_DIR,
check=True,
capture_output=True,
text=True,
)
for line in (result.stdout + result.stderr).splitlines():
print(f"> {line}")
return output_file
def run(self) -> None:
@@ -230,8 +259,10 @@ class PostImporter:
slug = post_data.get('slug') or slugify(title)
post_type = post_data.get('type') or 'post'
folder = f"({post_type}s)/{slug}"
bucket = f"{BLOG_PREFIX}/{uuid}/"
output_relative = f"src/routes/blog/{folder}/+page.md"
print(f"Importing:\n ID: {uuid}\n Title: {title}\n Path: routes/blog/{folder}/+page.md\n")
print(f"Importing:\n ID: {uuid}\n Title: {title}\n Path: {output_relative}\n Bucket: {bucket}")
print("Continue? [Y/n] (continuing in 5s) ", end="", flush=True)
ready, _, _ = select.select([sys.stdin], [], [], 5)
answer = sys.stdin.readline().strip().lower() if ready else ""
@@ -239,7 +270,8 @@ class PostImporter:
if answer.startswith("n"):
return
self.clear_s3_directory(f"{BLOG_PREFIX}/{uuid}/")
self.existing_keys = self.list_s3_keys(f"{bucket}")
with ImageProcessingRenderer(self, uuid, first_as_cover=post_type != 'release') as renderer:
doc = mistletoe.Document(post_data.content)
@@ -257,10 +289,16 @@ class PostImporter:
post_data['coverUrl'] = renderer.cover_image.src
post_data['coverAlt'] = "".join(child.content for child in getattr(renderer.cover_image, "children", []) if hasattr(child, "content"))
uploaded = len(self.referenced_keys - self.existing_keys)
unchanged = len(self.referenced_keys & self.existing_keys)
deleted = len(self.existing_keys - self.referenced_keys)
print(f"\nBucket stats (uploaded={uploaded}, unchanged={unchanged}, deleted={deleted})")
self.delete_unreferenced()
output_file = self.write_output(post_data, folder)
print(f"\nPost written to: {output_file}")
print(f"Post slug: {slug}")
print(f"\nhttp://localhost:5173/blog/{slug}")
if output := os.environ.get('GITHUB_OUTPUT'):
with open(output, 'a') as o:
@@ -268,4 +306,8 @@ class PostImporter:
o.write(f"uuid={uuid}\n")
PostImporter().run()
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <outline-post-url>", file=sys.stderr)
sys.exit(1)
PostImporter(sys.argv[1]).run()