mirror of
https://github.com/immich-app/yucca.git
synced 2026-09-30 13:33:00 +08:00
chore: remove the unused restic-api reference implementation (#684)
This commit is contained in:
Antoine Lecompte
@@ -1,671 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Benchmark restic backends (restic-api, michael, or both)"
|
|
||||||
#MISE depends=["docker:start", "restic-api:build", "michael:build"]
|
|
||||||
#MISE dir="{{config_root}}"
|
|
||||||
#
|
|
||||||
# Benchmark restic-compatible backends using the real restic CLI.
|
|
||||||
#
|
|
||||||
# Supports restic-api (Node.js/NestJS) and michael (Go), with an optional
|
|
||||||
# comparison mode that runs both sequentially and prints a side-by-side table.
|
|
||||||
#
|
|
||||||
# Prerequisites:
|
|
||||||
# - MinIO running (mise docker:start)
|
|
||||||
# - restic, node, perl installed (mise install)
|
|
||||||
# - For --compare: pnpm deps + go (mise install:deps)
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# # Benchmark both backends and compare (default):
|
|
||||||
# mise bench
|
|
||||||
#
|
|
||||||
# # Benchmark a single backend:
|
|
||||||
# mise bench -- --backend restic-api
|
|
||||||
# mise bench -- --backend michael
|
|
||||||
#
|
|
||||||
# Options:
|
|
||||||
# --backend NAME Benchmark only one backend: restic-api | michael
|
|
||||||
# --small-files N Number of small files (default: 1000)
|
|
||||||
# --medium-files N Number of medium files (default: 1000)
|
|
||||||
# --large-files N Number of large files (default: 100)
|
|
||||||
# --large-size-mb N Size of each large file (default: 100)
|
|
||||||
# --no-cleanup Keep S3 buckets after run
|
|
||||||
#
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
REPO_ROOT="$PWD"
|
|
||||||
|
|
||||||
# ─── Configuration ──────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
RESTIC_ENDPOINT="${RESTIC_ENDPOINT:-http://localhost:3010}"
|
|
||||||
# bench spawns + port-manages the backend, so split the endpoint back into host/port
|
|
||||||
_endpoint_hostport="${RESTIC_ENDPOINT#*://}"
|
|
||||||
API_HOST="${_endpoint_hostport%%:*}"
|
|
||||||
API_PORT="${_endpoint_hostport##*:}"
|
|
||||||
API_PORT="${API_PORT%%/*}"
|
|
||||||
# the restic-api / michael server bench launches binds RESTIC_API_PORT
|
|
||||||
export RESTIC_API_PORT="$API_PORT"
|
|
||||||
export JWT_SECRET="${JWT_SECRET:-cca13c34b450a77c1d4b9ecd25dff6aebc6d7417afdb31864f5943c59abd03a1}"
|
|
||||||
|
|
||||||
BACKEND=""
|
|
||||||
COMPARE=true
|
|
||||||
SMALL_COUNT=50
|
|
||||||
SMALL_SIZE_KB=100
|
|
||||||
MEDIUM_COUNT=50
|
|
||||||
MEDIUM_SIZE_MB=20
|
|
||||||
LARGE_COUNT=20
|
|
||||||
LARGE_SIZE_MB=100
|
|
||||||
DO_CLEANUP=true
|
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case "$1" in
|
|
||||||
--backend) BACKEND="$2"; COMPARE=false; shift 2 ;;
|
|
||||||
--small-files) SMALL_COUNT="$2"; shift 2 ;;
|
|
||||||
--medium-files) MEDIUM_COUNT="$2"; shift 2 ;;
|
|
||||||
--large-files) LARGE_COUNT="$2"; shift 2 ;;
|
|
||||||
--large-size-mb) LARGE_SIZE_MB="$2"; shift 2 ;;
|
|
||||||
--no-cleanup) DO_CLEANUP=false; shift ;;
|
|
||||||
*) echo "Unknown option: $1"; exit 1 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
BENCH_DIR=$(mktemp -d)
|
|
||||||
MEM_PID=""
|
|
||||||
API_PID=""
|
|
||||||
MANAGED_PID=""
|
|
||||||
REPOS=()
|
|
||||||
|
|
||||||
# ─── Formatting ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
bold() { printf '\033[1m%s\033[0m' "$*"; }
|
|
||||||
dim() { printf '\033[2m%s\033[0m' "$*"; }
|
|
||||||
green() { printf '\033[32m%s\033[0m' "$*"; }
|
|
||||||
red() { printf '\033[31m%s\033[0m' "$*"; }
|
|
||||||
|
|
||||||
# ─── Helpers ────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
on_exit() {
|
|
||||||
stop_mem_monitor
|
|
||||||
stop_backend
|
|
||||||
if [[ "$DO_CLEANUP" == true ]] && (( ${#REPOS[@]} > 0 )); then
|
|
||||||
cleanup_repos
|
|
||||||
fi
|
|
||||||
rm -rf "$BENCH_DIR"
|
|
||||||
}
|
|
||||||
trap on_exit EXIT
|
|
||||||
|
|
||||||
now_ms() {
|
|
||||||
perl -MTime::HiRes -e 'printf "%.0f\n", Time::HiRes::time() * 1000'
|
|
||||||
}
|
|
||||||
|
|
||||||
elapsed_s() {
|
|
||||||
perl -e "printf '%.3f', ($2 - $1) / 1000"
|
|
||||||
}
|
|
||||||
|
|
||||||
throughput_mbs() {
|
|
||||||
local bytes="$1" secs="$2"
|
|
||||||
perl -e "if ($secs > 0) { printf '%.1f', $bytes / 1048576 / $secs } else { print 'inf' }"
|
|
||||||
}
|
|
||||||
|
|
||||||
human_size() {
|
|
||||||
perl -e '
|
|
||||||
my $b = $ARGV[0];
|
|
||||||
if ($b >= 1073741824) { printf "%.1f GB", $b/1073741824 }
|
|
||||||
elsif ($b >= 1048576) { printf "%.1f MB", $b/1048576 }
|
|
||||||
elsif ($b >= 1024) { printf "%.1f KB", $b/1024 }
|
|
||||||
else { printf "%d B", $b }
|
|
||||||
' "$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
make_jwt() {
|
|
||||||
local repo="$1"
|
|
||||||
node --no-warnings -e "
|
|
||||||
const c = require('node:crypto');
|
|
||||||
const h = Buffer.from(JSON.stringify({alg:'HS256',typ:'JWT'})).toString('base64url');
|
|
||||||
const p = Buffer.from(JSON.stringify({
|
|
||||||
user: c.randomUUID(),
|
|
||||||
repository: process.argv[1],
|
|
||||||
writeOnce: false,
|
|
||||||
iat: Math.floor(Date.now()/1000),
|
|
||||||
exp: Math.floor(Date.now()/1000) + 7200
|
|
||||||
})).toString('base64url');
|
|
||||||
const s = c.createHmac('sha256', process.env.JWT_SECRET)
|
|
||||||
.update(h+'.'+p).digest('base64url');
|
|
||||||
process.stdout.write(h+'.'+p+'.'+s);
|
|
||||||
" "$repo"
|
|
||||||
}
|
|
||||||
|
|
||||||
init_repo() {
|
|
||||||
local repo
|
|
||||||
repo=$(node --no-warnings -e "process.stdout.write(require('node:crypto').randomUUID())")
|
|
||||||
local token
|
|
||||||
token=$(make_jwt "$repo")
|
|
||||||
REPOS+=("$repo")
|
|
||||||
|
|
||||||
export RESTIC_REPOSITORY="rest:http://_:${token}@${API_HOST}:${API_PORT}/${repo}"
|
|
||||||
export RESTIC_PASSWORD="bench"
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup_repos() {
|
|
||||||
echo ""
|
|
||||||
echo " $(dim "Cleaning up ${#REPOS[@]} benchmark repositories...")"
|
|
||||||
(cd "$REPO_ROOT/packages/restic-api" && node --no-warnings -e "
|
|
||||||
const { S3Client, ListObjectsV2Command, DeleteObjectsCommand, DeleteBucketCommand }
|
|
||||||
= require('@aws-sdk/client-s3');
|
|
||||||
const client = new S3Client({
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: process.env.S3_ACCESS_KEY_ID || 'minio',
|
|
||||||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || 'miniominio',
|
|
||||||
},
|
|
||||||
region: process.env.S3_REGION || 'minio',
|
|
||||||
endpoint: process.env.S3_ENDPOINT || 'http://localhost:9000',
|
|
||||||
forcePathStyle: true,
|
|
||||||
});
|
|
||||||
async function nuke(Bucket) {
|
|
||||||
try {
|
|
||||||
const { Contents } = await client.send(new ListObjectsV2Command({ Bucket }));
|
|
||||||
if (Contents?.length) {
|
|
||||||
await client.send(new DeleteObjectsCommand({
|
|
||||||
Bucket, Delete: { Objects: Contents.map(({ Key }) => ({ Key })) }
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
await client.send(new DeleteBucketCommand({ Bucket }));
|
|
||||||
} catch {}
|
|
||||||
}
|
|
||||||
Promise.all(process.argv.slice(1).map(nuke)).then(() => process.exit(0));
|
|
||||||
" "${REPOS[@]}") || true
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Results I/O ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
save_result() {
|
|
||||||
echo "$1=$2" >> "$BENCH_DIR/results/${CURRENT_BACKEND}.txt"
|
|
||||||
}
|
|
||||||
|
|
||||||
read_result() {
|
|
||||||
local key="$1" file="$2"
|
|
||||||
grep "^${key}=" "$file" 2>/dev/null | tail -1 | cut -d= -f2
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Memory monitoring ──────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
start_mem_monitor() {
|
|
||||||
API_PID=$(lsof -ti ":$API_PORT" -sTCP:LISTEN 2>/dev/null | head -1 || true)
|
|
||||||
if [[ -z "$API_PID" ]]; then
|
|
||||||
echo " $(dim '(could not find API process — memory stats unavailable)')"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
local peak_file="$BENCH_DIR/.mem_peak"
|
|
||||||
echo "0" > "$peak_file"
|
|
||||||
(
|
|
||||||
while kill -0 "$API_PID" 2>/dev/null; do
|
|
||||||
rss_kb=$(ps -o rss= -p "$API_PID" 2>/dev/null | tr -d ' ' || echo 0)
|
|
||||||
peak=$(cat "$peak_file")
|
|
||||||
if (( rss_kb > peak )); then echo "$rss_kb" > "$peak_file"; fi
|
|
||||||
sleep 0.5
|
|
||||||
done
|
|
||||||
) &
|
|
||||||
MEM_PID=$!
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_mem_monitor() {
|
|
||||||
[[ -n "${MEM_PID:-}" ]] && kill "$MEM_PID" 2>/dev/null || true
|
|
||||||
MEM_PID=""
|
|
||||||
API_PID=""
|
|
||||||
}
|
|
||||||
|
|
||||||
mem_rss_kb() {
|
|
||||||
[[ -z "${API_PID:-}" ]] && echo 0 && return
|
|
||||||
ps -o rss= -p "$API_PID" 2>/dev/null | tr -d ' ' || echo 0
|
|
||||||
}
|
|
||||||
|
|
||||||
mem_peak_kb() {
|
|
||||||
local peak_file="$BENCH_DIR/.mem_peak"
|
|
||||||
[[ -f "$peak_file" ]] && cat "$peak_file" || echo 0
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt_mem_mb() {
|
|
||||||
perl -e "printf '%.1f MB', $1 / 1024"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Backend lifecycle ──────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
start_backend() {
|
|
||||||
local name="$1"
|
|
||||||
echo " $(dim "Starting ${name}...")"
|
|
||||||
|
|
||||||
# Ensure port is free before starting
|
|
||||||
lsof -ti ":$API_PORT" -sTCP:LISTEN 2>/dev/null | xargs kill 2>/dev/null || true
|
|
||||||
local elapsed=0
|
|
||||||
while nc -z "$API_HOST" "$API_PORT" 2>/dev/null; do
|
|
||||||
sleep 0.2
|
|
||||||
elapsed=$((elapsed + 1))
|
|
||||||
if (( elapsed > 50 )); then
|
|
||||||
echo "ERROR: port $API_PORT still in use after 10 seconds"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
case "$name" in
|
|
||||||
restic-api)
|
|
||||||
(cd "$REPO_ROOT" && exec pnpm --filter restic-api start 2>&1) > "$BENCH_DIR/backend.log" 2>&1 &
|
|
||||||
MANAGED_PID=$!
|
|
||||||
;;
|
|
||||||
michael)
|
|
||||||
(cd "$REPO_ROOT/packages/michael" && \
|
|
||||||
OTLP_METRICS_ENDPOINT=localhost:8428 \
|
|
||||||
OTLP_METRICS_URL_PATH=/opentelemetry/api/v1/push \
|
|
||||||
exec go run . 2>&1) > "$BENCH_DIR/backend.log" 2>&1 &
|
|
||||||
MANAGED_PID=$!
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "ERROR: unknown backend '$name' (expected restic-api or michael)"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
local elapsed=0
|
|
||||||
while ! nc -z "$API_HOST" "$API_PORT" 2>/dev/null; do
|
|
||||||
sleep 0.2
|
|
||||||
elapsed=$((elapsed + 1))
|
|
||||||
if (( elapsed > 150 )); then
|
|
||||||
echo "ERROR: $name did not start within 30 seconds"
|
|
||||||
echo "--- backend log ---"
|
|
||||||
cat "$BENCH_DIR/backend.log"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo " $(dim "${name} ready on :${API_PORT}")"
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_backend() {
|
|
||||||
if [[ -n "${MANAGED_PID:-}" ]]; then
|
|
||||||
kill "$MANAGED_PID" 2>/dev/null || true
|
|
||||||
wait "$MANAGED_PID" 2>/dev/null || true
|
|
||||||
MANAGED_PID=""
|
|
||||||
fi
|
|
||||||
# Also kill anything still on the port (child processes)
|
|
||||||
lsof -ti ":$API_PORT" -sTCP:LISTEN 2>/dev/null | xargs kill 2>/dev/null || true
|
|
||||||
|
|
||||||
# Wait for port to free up
|
|
||||||
local elapsed=0
|
|
||||||
while nc -z "$API_HOST" "$API_PORT" 2>/dev/null; do
|
|
||||||
sleep 0.2
|
|
||||||
elapsed=$((elapsed + 1))
|
|
||||||
if (( elapsed > 50 )); then break; fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Prerequisites ──────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
check_prereqs() {
|
|
||||||
local missing=()
|
|
||||||
command -v restic >/dev/null 2>&1 || missing+=("restic")
|
|
||||||
command -v node >/dev/null 2>&1 || missing+=("node")
|
|
||||||
command -v perl >/dev/null 2>&1 || missing+=("perl")
|
|
||||||
|
|
||||||
if [[ "$COMPARE" == true ]]; then
|
|
||||||
command -v pnpm >/dev/null 2>&1 || missing+=("pnpm")
|
|
||||||
command -v go >/dev/null 2>&1 || missing+=("go")
|
|
||||||
fi
|
|
||||||
|
|
||||||
if (( ${#missing[@]} > 0 )); then
|
|
||||||
echo "ERROR: missing required tools: ${missing[*]}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
check_api_reachable() {
|
|
||||||
local http_code
|
|
||||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 \
|
|
||||||
"http://${API_HOST}:${API_PORT}/" 2>/dev/null || echo "000")
|
|
||||||
if [[ "$http_code" == "000" ]]; then
|
|
||||||
echo "ERROR: backend not reachable at http://${API_HOST}:${API_PORT}"
|
|
||||||
echo " Start it with: mise restic-api:dev or mise michael:dev"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Data generation ────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
generate_data() {
|
|
||||||
echo "$(bold 'Generating test data...')"
|
|
||||||
|
|
||||||
local dir="$BENCH_DIR/data/small" && mkdir -p "$dir"
|
|
||||||
for i in $(seq 1 "$SMALL_COUNT"); do
|
|
||||||
dd if=/dev/urandom of="$dir/f_$i" bs=1024 count="$SMALL_SIZE_KB" 2>/dev/null
|
|
||||||
done
|
|
||||||
echo " small: ${SMALL_COUNT} files x ${SMALL_SIZE_KB} KB"
|
|
||||||
|
|
||||||
dir="$BENCH_DIR/data/medium" && mkdir -p "$dir"
|
|
||||||
for i in $(seq 1 "$MEDIUM_COUNT"); do
|
|
||||||
dd if=/dev/urandom of="$dir/f_$i" bs=1048576 count="$MEDIUM_SIZE_MB" 2>/dev/null
|
|
||||||
done
|
|
||||||
echo " medium: ${MEDIUM_COUNT} files x ${MEDIUM_SIZE_MB} MB"
|
|
||||||
|
|
||||||
dir="$BENCH_DIR/data/large" && mkdir -p "$dir"
|
|
||||||
for i in $(seq 1 "$LARGE_COUNT"); do
|
|
||||||
dd if=/dev/urandom of="$dir/f_$i" bs=1048576 count="$LARGE_SIZE_MB" 2>/dev/null
|
|
||||||
done
|
|
||||||
echo " large: ${LARGE_COUNT} files x ${LARGE_SIZE_MB} MB"
|
|
||||||
|
|
||||||
local total_bytes=$(( SMALL_COUNT * SMALL_SIZE_KB * 1024 + MEDIUM_COUNT * MEDIUM_SIZE_MB * 1048576 + LARGE_COUNT * LARGE_SIZE_MB * 1048576 ))
|
|
||||||
local peak_bytes=$(( total_bytes * 3 ))
|
|
||||||
echo ""
|
|
||||||
echo " $(bold "total: $(human_size $total_bytes)") $(dim "(~$(human_size $peak_bytes) peak disk + S3)")"
|
|
||||||
echo ""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Throughput scenario ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
# run_throughput <tag> <label> <data_dir> <total_bytes> <file_count>
|
|
||||||
run_throughput() {
|
|
||||||
local tag="$1" label="$2" data_dir="$3" total_bytes="$4" file_count="$5"
|
|
||||||
local restic_log="$BENCH_DIR/restic_${tag}.log"
|
|
||||||
|
|
||||||
echo "$(bold " $label")"
|
|
||||||
|
|
||||||
init_repo
|
|
||||||
|
|
||||||
# init
|
|
||||||
local t0 t1
|
|
||||||
t0=$(now_ms)
|
|
||||||
if ! restic init -q 2>"$restic_log"; then
|
|
||||||
echo " $(red 'ERROR: restic init failed')"
|
|
||||||
cat "$restic_log" | sed 's/^/ /' >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
t1=$(now_ms)
|
|
||||||
echo " init: $(elapsed_s "$t0" "$t1")s"
|
|
||||||
|
|
||||||
# backup
|
|
||||||
t0=$(now_ms)
|
|
||||||
if ! restic backup -q "$data_dir" 2>"$restic_log"; then
|
|
||||||
echo " $(red 'ERROR: restic backup failed')"
|
|
||||||
cat "$restic_log" | sed 's/^/ /' >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
t1=$(now_ms)
|
|
||||||
local backup_s backup_tp
|
|
||||||
backup_s=$(elapsed_s "$t0" "$t1")
|
|
||||||
backup_tp=$(throughput_mbs "$total_bytes" "$backup_s")
|
|
||||||
echo " backup: ${backup_s}s @ ${backup_tp} MB/s"
|
|
||||||
save_result "${tag}_backup_s" "$backup_s"
|
|
||||||
save_result "${tag}_backup_mbs" "$backup_tp"
|
|
||||||
|
|
||||||
if (( file_count > 1 )); then
|
|
||||||
local ms_per_file
|
|
||||||
ms_per_file=$(perl -e "printf '%.1f', ($t1 - $t0) / $file_count")
|
|
||||||
echo " $(dim " ~${ms_per_file} ms/file")"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# restore
|
|
||||||
local restore_dir="$BENCH_DIR/restore_${RANDOM}"
|
|
||||||
mkdir -p "$restore_dir"
|
|
||||||
t0=$(now_ms)
|
|
||||||
if ! restic restore latest --target "$restore_dir" -q 2>"$restic_log"; then
|
|
||||||
echo " $(red 'ERROR: restic restore failed')"
|
|
||||||
cat "$restic_log" | sed 's/^/ /' >&2
|
|
||||||
rm -rf "$restore_dir"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
t1=$(now_ms)
|
|
||||||
local restore_s restore_tp
|
|
||||||
restore_s=$(elapsed_s "$t0" "$t1")
|
|
||||||
restore_tp=$(throughput_mbs "$total_bytes" "$restore_s")
|
|
||||||
echo " restore: ${restore_s}s @ ${restore_tp} MB/s"
|
|
||||||
save_result "${tag}_restore_s" "$restore_s"
|
|
||||||
save_result "${tag}_restore_mbs" "$restore_tp"
|
|
||||||
rm -rf "$restore_dir"
|
|
||||||
|
|
||||||
# incremental (re-backup identical data — exercises index/dedup path)
|
|
||||||
t0=$(now_ms)
|
|
||||||
if ! restic backup -q "$data_dir" 2>"$restic_log"; then
|
|
||||||
echo " $(red 'ERROR: restic incremental backup failed')"
|
|
||||||
cat "$restic_log" | sed 's/^/ /' >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
t1=$(now_ms)
|
|
||||||
local incr_s
|
|
||||||
incr_s=$(elapsed_s "$t0" "$t1")
|
|
||||||
echo " incremental: ${incr_s}s (no new data)"
|
|
||||||
save_result "${tag}_incr_s" "$incr_s"
|
|
||||||
|
|
||||||
# memory
|
|
||||||
if [[ -n "${API_PID:-}" ]]; then
|
|
||||||
echo " memory: $(fmt_mem_mb "$(mem_rss_kb)") current, $(fmt_mem_mb "$(mem_peak_kb)") peak"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Run full benchmark suite for one backend ──────────────────────────────────
|
|
||||||
|
|
||||||
CURRENT_BACKEND=""
|
|
||||||
|
|
||||||
run_suite() {
|
|
||||||
local backend="$1"
|
|
||||||
CURRENT_BACKEND="$backend"
|
|
||||||
mkdir -p "$BENCH_DIR/results"
|
|
||||||
: > "$BENCH_DIR/results/${backend}.txt"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "$(bold '════════════════════════════════════════════════════════════')"
|
|
||||||
echo "$(bold " ${backend}")"
|
|
||||||
echo "$(bold '════════════════════════════════════════════════════════════')"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# memory baseline
|
|
||||||
stop_mem_monitor
|
|
||||||
echo "0" > "$BENCH_DIR/.mem_peak"
|
|
||||||
start_mem_monitor
|
|
||||||
local baseline_kb
|
|
||||||
baseline_kb=$(mem_rss_kb)
|
|
||||||
save_result "mem_baseline_kb" "$baseline_kb"
|
|
||||||
echo "$(bold 'Memory')"
|
|
||||||
echo " baseline: $(fmt_mem_mb "$baseline_kb") (PID ${API_PID:-?})"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
local small_bytes=$((SMALL_COUNT * SMALL_SIZE_KB * 1024))
|
|
||||||
local medium_bytes=$((MEDIUM_COUNT * MEDIUM_SIZE_MB * 1048576))
|
|
||||||
local large_bytes=$((LARGE_COUNT * LARGE_SIZE_MB * 1048576))
|
|
||||||
|
|
||||||
echo "$(bold 'Throughput')"
|
|
||||||
echo ""
|
|
||||||
run_throughput "small" \
|
|
||||||
"small (${SMALL_COUNT} x ${SMALL_SIZE_KB} KB = $(human_size $small_bytes))" \
|
|
||||||
"$BENCH_DIR/data/small" "$small_bytes" "$SMALL_COUNT"
|
|
||||||
|
|
||||||
run_throughput "medium" \
|
|
||||||
"medium (${MEDIUM_COUNT} x ${MEDIUM_SIZE_MB} MB = $(human_size $medium_bytes))" \
|
|
||||||
"$BENCH_DIR/data/medium" "$medium_bytes" "$MEDIUM_COUNT"
|
|
||||||
|
|
||||||
run_throughput "large" \
|
|
||||||
"large (${LARGE_COUNT} x ${LARGE_SIZE_MB} MB = $(human_size $large_bytes))" \
|
|
||||||
"$BENCH_DIR/data/large" "$large_bytes" "$LARGE_COUNT"
|
|
||||||
|
|
||||||
# save final memory
|
|
||||||
save_result "mem_peak_kb" "$(mem_peak_kb)"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Print single-backend summary ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
print_summary() {
|
|
||||||
local f="$BENCH_DIR/results/$1.txt"
|
|
||||||
|
|
||||||
echo "$(bold '════════════════════════════════════════════════════════════')"
|
|
||||||
echo "$(bold "Summary: $1")"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
printf ' %-28s %10s %10s %10s %10s\n' "" "backup" "restore" "up MB/s" "down MB/s"
|
|
||||||
printf ' %-28s %10s %10s %10s %10s\n' "" "──────" "───────" "───────" "─────────"
|
|
||||||
for tag in small medium large; do
|
|
||||||
printf ' %-28s %9ss %9ss %9s %9s\n' \
|
|
||||||
"$tag" \
|
|
||||||
"$(read_result "${tag}_backup_s" "$f")" \
|
|
||||||
"$(read_result "${tag}_restore_s" "$f")" \
|
|
||||||
"$(read_result "${tag}_backup_mbs" "$f")" \
|
|
||||||
"$(read_result "${tag}_restore_mbs" "$f")"
|
|
||||||
done
|
|
||||||
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
local peak_kb
|
|
||||||
peak_kb=$(read_result mem_peak_kb "$f")
|
|
||||||
local base_kb
|
|
||||||
base_kb=$(read_result mem_baseline_kb "$f")
|
|
||||||
echo " memory: baseline $(fmt_mem_mb "$base_kb"), peak $(fmt_mem_mb "$peak_kb")"
|
|
||||||
|
|
||||||
if [[ "$DO_CLEANUP" == false ]]; then
|
|
||||||
echo ""
|
|
||||||
echo " $(dim "Repos kept (--no-cleanup): ${REPOS[*]}")"
|
|
||||||
fi
|
|
||||||
echo ""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Print comparison table ────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
# pct_delta <a> <b> → prints "(+12.3%)" or "(-12.3%)"
|
|
||||||
# positive means b > a
|
|
||||||
pct_delta() {
|
|
||||||
perl -e '
|
|
||||||
my ($a, $b) = @ARGV;
|
|
||||||
if ($a == 0) { print "n/a"; exit }
|
|
||||||
my $d = ($b - $a) / abs($a) * 100;
|
|
||||||
printf "%+.1f%%", $d;
|
|
||||||
' "$1" "$2"
|
|
||||||
}
|
|
||||||
|
|
||||||
# color_delta <a> <b> <higher_is_better>
|
|
||||||
# prints the delta in green if improvement, red if regression
|
|
||||||
color_delta() {
|
|
||||||
local a="$1" b="$2" higher_better="${3:-false}"
|
|
||||||
local raw
|
|
||||||
raw=$(pct_delta "$a" "$b")
|
|
||||||
[[ "$raw" == "n/a" ]] && printf '%s' "$raw" && return
|
|
||||||
|
|
||||||
local numeric
|
|
||||||
numeric=$(perl -e '
|
|
||||||
my ($a, $b) = @ARGV;
|
|
||||||
if ($a == 0) { print 0; exit }
|
|
||||||
printf "%.1f", ($b - $a) / abs($a) * 100;
|
|
||||||
' "$a" "$b")
|
|
||||||
|
|
||||||
local is_better
|
|
||||||
if [[ "$higher_better" == true ]]; then
|
|
||||||
is_better=$(perl -e "print $numeric > 0 ? 1 : 0")
|
|
||||||
else
|
|
||||||
is_better=$(perl -e "print $numeric < 0 ? 1 : 0")
|
|
||||||
fi
|
|
||||||
|
|
||||||
if (( is_better )); then
|
|
||||||
green "$raw"
|
|
||||||
else
|
|
||||||
# within 5% = dim, otherwise red
|
|
||||||
local abs_val
|
|
||||||
abs_val=$(perl -e "printf '%.1f', abs($numeric)")
|
|
||||||
if perl -e "exit( $abs_val < 5.0 ? 0 : 1 )"; then
|
|
||||||
dim "$raw"
|
|
||||||
else
|
|
||||||
red "$raw"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
print_comparison() {
|
|
||||||
local fa="$BENCH_DIR/results/restic-api.txt"
|
|
||||||
local fb="$BENCH_DIR/results/michael.txt"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "$(bold '════════════════════════════════════════════════════════════════════════════════')"
|
|
||||||
echo "$(bold 'Comparison: restic-api vs michael')"
|
|
||||||
echo "$(bold '════════════════════════════════════════════════════════════════════════════════')"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Header
|
|
||||||
printf ' %-24s %16s %16s %10s\n' "" "restic-api" "michael" "delta"
|
|
||||||
printf ' %-24s %16s %16s %10s\n' "" "──────────" "───────" "─────"
|
|
||||||
|
|
||||||
# Throughput rows — show MB/s (higher is better)
|
|
||||||
for tag in small medium large; do
|
|
||||||
for op in backup restore; do
|
|
||||||
local key="${tag}_${op}_mbs"
|
|
||||||
local va vb
|
|
||||||
va=$(read_result "$key" "$fa")
|
|
||||||
vb=$(read_result "$key" "$fb")
|
|
||||||
local delta
|
|
||||||
delta=$(color_delta "$va" "$vb" true)
|
|
||||||
printf ' %-24s %12s MB/s %12s MB/s %b\n' \
|
|
||||||
"${tag} ${op}" "$va" "$vb" "$delta"
|
|
||||||
done
|
|
||||||
|
|
||||||
local key="${tag}_incr_s"
|
|
||||||
local va vb
|
|
||||||
va=$(read_result "$key" "$fa")
|
|
||||||
vb=$(read_result "$key" "$fb")
|
|
||||||
local delta
|
|
||||||
delta=$(color_delta "$va" "$vb" false)
|
|
||||||
printf ' %-24s %15ss %15ss %b\n' \
|
|
||||||
"${tag} incremental" "$va" "$vb" "$delta"
|
|
||||||
done
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Memory — lower is better
|
|
||||||
local va vb delta
|
|
||||||
va=$(read_result mem_baseline_kb "$fa")
|
|
||||||
vb=$(read_result mem_baseline_kb "$fb")
|
|
||||||
delta=$(color_delta "$va" "$vb" false)
|
|
||||||
printf ' %-24s %16s %16s %b\n' \
|
|
||||||
"memory baseline" "$(fmt_mem_mb "$va")" "$(fmt_mem_mb "$vb")" "$delta"
|
|
||||||
|
|
||||||
va=$(read_result mem_peak_kb "$fa")
|
|
||||||
vb=$(read_result mem_peak_kb "$fb")
|
|
||||||
delta=$(color_delta "$va" "$vb" false)
|
|
||||||
printf ' %-24s %16s %16s %b\n' \
|
|
||||||
"memory peak" "$(fmt_mem_mb "$va")" "$(fmt_mem_mb "$vb")" "$delta"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─── Main ───────────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
main() {
|
|
||||||
echo ""
|
|
||||||
echo "$(bold 'restic backend benchmark')"
|
|
||||||
echo "$(dim '────────────────────────────────────────────────────────────')"
|
|
||||||
echo " restic $(restic version 2>&1 | head -1)"
|
|
||||||
echo " node $(node -v)"
|
|
||||||
if [[ "$COMPARE" == true ]]; then
|
|
||||||
echo " go $(go version 2>&1 | awk '{print $3}')"
|
|
||||||
echo " mode comparison (restic-api vs michael)"
|
|
||||||
else
|
|
||||||
echo " backend ${BACKEND}"
|
|
||||||
fi
|
|
||||||
echo " api http://${API_HOST}:${API_PORT}"
|
|
||||||
echo " tmpdir ${BENCH_DIR}"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
check_prereqs
|
|
||||||
generate_data
|
|
||||||
|
|
||||||
if [[ "$COMPARE" == true ]]; then
|
|
||||||
for backend in restic-api michael; do
|
|
||||||
start_backend "$backend"
|
|
||||||
run_suite "$backend"
|
|
||||||
stop_mem_monitor
|
|
||||||
stop_backend
|
|
||||||
done
|
|
||||||
print_comparison
|
|
||||||
else
|
|
||||||
if nc -z "$API_HOST" "$API_PORT" 2>/dev/null; then
|
|
||||||
echo " $(dim "Using already-running backend on :${API_PORT}")"
|
|
||||||
else
|
|
||||||
start_backend "$BACKEND"
|
|
||||||
fi
|
|
||||||
run_suite "$BACKEND"
|
|
||||||
stop_mem_monitor
|
|
||||||
stop_backend
|
|
||||||
print_summary "$BACKEND"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
#MISE dir="{{config_root}}/packages/michael"
|
#MISE dir="{{config_root}}/packages/michael"
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
# Complements the fleet-level `mise bench` (real restic CLI against a running
|
# Complements `yuctl tools bench` (real restic CLI against a running
|
||||||
# backend): these measure per-request cost inside the process.
|
# backend): these measure per-request cost inside the process.
|
||||||
#
|
#
|
||||||
# The source-network benchmarks resolve against the committed three-network
|
# The source-network benchmarks resolve against the committed three-network
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#MISE description="Run michael in development mode"
|
#MISE description="Run michael in development mode"
|
||||||
set -e
|
set -e
|
||||||
source "$(dirname "$0")/../restic-api/env"
|
source "$(dirname "$0")/env"
|
||||||
|
|
||||||
# No load balancer in front of a local run, so there is nothing to drain away
|
# No load balancer in front of a local run, so there is nothing to drain away
|
||||||
# from — skip the fleet's 15s SIGTERM drain and let Ctrl-C exit immediately.
|
# from — skip the fleet's 15s SIGTERM drain and let Ctrl-C exit immediately.
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Build restic-api"
|
|
||||||
#MISE depends=["common:build"]
|
|
||||||
set -e
|
|
||||||
|
|
||||||
NODE_ENV=production pnpm --filter restic-api build
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Run restic-api in development mode"
|
|
||||||
set -e
|
|
||||||
source "$(dirname "$0")/env"
|
|
||||||
|
|
||||||
pnpm --filter restic-api start:dev
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Run unit tests for restic-api"
|
|
||||||
set -e
|
|
||||||
source "$(dirname "$0")/../env"
|
|
||||||
|
|
||||||
pnpm --filter restic-api test "$@"
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Run integration tests for restic-api"
|
|
||||||
set -e
|
|
||||||
source "$(dirname "$0")/../env"
|
|
||||||
|
|
||||||
pnpm --filter restic-api test:integration "$@"
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#MISE description="Run unit tests in watch mode for restic-api"
|
|
||||||
set -e
|
|
||||||
source "$(dirname "$0")/../env"
|
|
||||||
|
|
||||||
pnpm --filter restic-api test --watch "$@"
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
#MISE description="Run all end-to-end tests"
|
#MISE description="Run all end-to-end tests"
|
||||||
#MISE depends=["test:e2e:wait"]
|
#MISE depends=["test:e2e:wait"]
|
||||||
set -e
|
set -e
|
||||||
source "$(dirname "$0")/../../restic-api/env"
|
source "$(dirname "$0")/../../michael/env"
|
||||||
source "$(dirname "$0")/../../yucca-api/env"
|
source "$(dirname "$0")/../../yucca-api/env"
|
||||||
|
|
||||||
pnpm --filter e2e test "$@"
|
pnpm --filter e2e test "$@"
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# Targets the compose/`mise dev` flow (the k3d flow has its own readiness
|
# Targets the compose/`mise dev` flow (the k3d flow has its own readiness
|
||||||
# handling in packages/e2e/k3d/run.sh and does not use this task).
|
# handling in packages/e2e/k3d/run.sh and does not use this task).
|
||||||
set -e
|
set -e
|
||||||
source "$(dirname "$0")/../../restic-api/env"
|
source "$(dirname "$0")/../../michael/env"
|
||||||
source "$(dirname "$0")/../../yucca-api/env"
|
source "$(dirname "$0")/../../yucca-api/env"
|
||||||
|
|
||||||
echo Ensure dev environment is running before invoking tests.
|
echo Ensure dev environment is running before invoking tests.
|
||||||
@@ -24,7 +24,7 @@ wait_for_port() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
wait_for_port 8092 mock-oidc-provider
|
wait_for_port 8092 mock-oidc-provider
|
||||||
wait_for_port "$RESTIC_API_PORT" restic-api
|
wait_for_port "$RESTIC_API_PORT" michael
|
||||||
wait_for_port "$YUCCA_API_PORT" yucca-api
|
wait_for_port "$YUCCA_API_PORT" yucca-api
|
||||||
wait_for_port 22676 orchestration-api
|
wait_for_port 22676 orchestration-api
|
||||||
wait_for_port "${WEB_PORT:-36033}" web
|
wait_for_port "${WEB_PORT:-36033}" web
|
||||||
|
|||||||
@@ -62,4 +62,4 @@ export RADOS_ENDPOINT RADOS_ACCESS_KEY_ID RADOS_SECRET_ACCESS_KEY
|
|||||||
|
|
||||||
echo "==> run the S3-backed integration suites"
|
echo "==> run the S3-backed integration suites"
|
||||||
# Concurrent: each scopes itself to freshly generated repository/bucket names.
|
# Concurrent: each scopes itself to freshly generated repository/bucket names.
|
||||||
mise run michael:test:integration ::: restic-api:test:integration ::: yucca-metrics-worker:test:integration
|
mise run michael:test:integration ::: yucca-metrics-worker:test:integration
|
||||||
|
|||||||
@@ -109,7 +109,6 @@ Zod-validated `env.ts`, JWT auth guards via `@AuthRoute()`, OTel from `@common/s
|
|||||||
| `michael` | Go | **Production** restic REST backend — S3 proxy with JWT verification, WORM enforcement, backend pooling. |
|
| `michael` | Go | **Production** restic REST backend — S3 proxy with JWT verification, WORM enforcement, backend pooling. |
|
||||||
| `columbo` | Go | Ticket investigation agent: LLM loop (OpenRouter) over per-user-scoped o11y queries, answers only into staff threads. See `docs/columbo.md`. |
|
| `columbo` | Go | Ticket investigation agent: LLM loop (OpenRouter) over per-user-scoped o11y queries, answers only into staff threads. See `docs/columbo.md`. |
|
||||||
| `monk` | Go | Ceph scrub-backlog exporter: polls `pg ls`, serves measured per-pool scrub-age metrics on :9284. Deployed onto mon hosts by ansible, not K8s. |
|
| `monk` | Go | Ceph scrub-backlog exporter: polls `pg ls`, serves measured per-pool scrub-age metrics on :9284. Deployed onto mon hosts by ansible, not K8s. |
|
||||||
| `restic-api` | NestJS | Earlier TS implementation of the restic backend, kept as **reference**; not deployed. |
|
|
||||||
| `yucca-metrics-worker` | NestJS | 5-min cron: RadosGW usage → meter tables → per-connection rollup (`connectionMetrics`, billing floor), OTel gauges. |
|
| `yucca-metrics-worker` | NestJS | 5-min cron: RadosGW usage → meter tables → per-connection rollup (`connectionMetrics`, billing floor), OTel gauges. |
|
||||||
| `redis` (valkey) | | Shared platform cache (ephemeral; keys `yucca:<service>:<purpose>:*`). Primary-region only. |
|
| `redis` (valkey) | | Shared platform cache (ephemeral; keys `yucca:<service>:<purpose>:*`). Primary-region only. |
|
||||||
| `mock-oidc-provider` | Node | Dev/test OIDC IdP (code + device flow). |
|
| `mock-oidc-provider` | Node | Dev/test OIDC IdP (code + device flow). |
|
||||||
|
|||||||
@@ -112,9 +112,9 @@ wait_for_http yucca-api http://localhost:3020/api/meta 200
|
|||||||
# call, and only answers once discovery has reached a live yucca-api.
|
# call, and only answers once discovery has reached a live yucca-api.
|
||||||
wait_for_http orchestration-api http://localhost:22676/api/yucca/onboarding 200
|
wait_for_http orchestration-api http://localhost:22676/api/yucca/onboarding 200
|
||||||
|
|
||||||
echo "==> jest e2e (it-works, restic-api, yucca-api, orchestration-api)"
|
echo "==> jest e2e (it-works, restic, yucca-api, orchestration-api)"
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source .mise/tasks/restic-api/env
|
source .mise/tasks/michael/env
|
||||||
# shellcheck disable=SC1091
|
# shellcheck disable=SC1091
|
||||||
source .mise/tasks/yucca-api/env
|
source .mise/tasks/yucca-api/env
|
||||||
# Three, not one per suite: orchestration-api alone runs ~105s and gates the
|
# Three, not one per suite: orchestration-api alone runs ~105s and gates the
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
# compiled output
|
|
||||||
/dist
|
|
||||||
/node_modules
|
|
||||||
/build
|
|
||||||
|
|
||||||
# Logs
|
|
||||||
logs
|
|
||||||
*.log
|
|
||||||
npm-debug.log*
|
|
||||||
pnpm-debug.log*
|
|
||||||
yarn-debug.log*
|
|
||||||
yarn-error.log*
|
|
||||||
lerna-debug.log*
|
|
||||||
|
|
||||||
# OS
|
|
||||||
.DS_Store
|
|
||||||
|
|
||||||
# Tests
|
|
||||||
/coverage
|
|
||||||
/.nyc_output
|
|
||||||
|
|
||||||
# IDEs and editors
|
|
||||||
/.idea
|
|
||||||
.project
|
|
||||||
.classpath
|
|
||||||
.c9/
|
|
||||||
*.launch
|
|
||||||
.settings/
|
|
||||||
*.sublime-workspace
|
|
||||||
|
|
||||||
# IDE - VSCode
|
|
||||||
.vscode/*
|
|
||||||
!.vscode/settings.json
|
|
||||||
!.vscode/tasks.json
|
|
||||||
!.vscode/launch.json
|
|
||||||
!.vscode/extensions.json
|
|
||||||
|
|
||||||
# dotenv environment variable files
|
|
||||||
.env
|
|
||||||
.env.development.local
|
|
||||||
.env.test.local
|
|
||||||
.env.production.local
|
|
||||||
.env.local
|
|
||||||
|
|
||||||
# temp directory
|
|
||||||
.temp
|
|
||||||
.tmp
|
|
||||||
|
|
||||||
# Runtime data
|
|
||||||
pids
|
|
||||||
*.pid
|
|
||||||
*.seed
|
|
||||||
*.pid.lock
|
|
||||||
|
|
||||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
|
||||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://json.schemastore.org/nest-cli",
|
|
||||||
"collection": "@nestjs/schematics",
|
|
||||||
"sourceRoot": "src",
|
|
||||||
"compilerOptions": {
|
|
||||||
"deleteOutDir": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "restic-api",
|
|
||||||
"version": "0.43.0",
|
|
||||||
"description": "",
|
|
||||||
"author": "",
|
|
||||||
"private": true,
|
|
||||||
"license": "UNLICENSED",
|
|
||||||
"files": [
|
|
||||||
"dist"
|
|
||||||
],
|
|
||||||
"scripts": {
|
|
||||||
"build": "nest build",
|
|
||||||
"start": "nest start",
|
|
||||||
"start:dev": "nest start --watch",
|
|
||||||
"start:debug": "nest start --debug --watch",
|
|
||||||
"start:prod": "node dist/main",
|
|
||||||
"test": "jest",
|
|
||||||
"test:watch": "jest --watch",
|
|
||||||
"test:cov": "jest --coverage",
|
|
||||||
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
|
|
||||||
"test:integration": "jest --config ./test/jest-integration.json"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@aws-sdk/client-s3": "catalog:",
|
|
||||||
"@aws-sdk/lib-storage": "catalog:",
|
|
||||||
"@common/server": "workspace:^",
|
|
||||||
"@nestjs/common": "catalog:",
|
|
||||||
"@nestjs/core": "catalog:",
|
|
||||||
"@nestjs/jwt": "catalog:",
|
|
||||||
"@nestjs/platform-express": "catalog:",
|
|
||||||
"@opentelemetry/api": "catalog:",
|
|
||||||
"nestjs-zod": "catalog:",
|
|
||||||
"reflect-metadata": "catalog:",
|
|
||||||
"rxjs": "catalog:",
|
|
||||||
"zod": "catalog:"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@nestjs/cli": "catalog:",
|
|
||||||
"@nestjs/schematics": "catalog:",
|
|
||||||
"@nestjs/testing": "catalog:",
|
|
||||||
"@types/express": "catalog:",
|
|
||||||
"@types/jest": "catalog:",
|
|
||||||
"@types/node": "catalog:",
|
|
||||||
"@types/supertest": "catalog:",
|
|
||||||
"globals": "catalog:",
|
|
||||||
"jest": "catalog:",
|
|
||||||
"source-map-support": "catalog:",
|
|
||||||
"supertest": "catalog:",
|
|
||||||
"ts-jest": "catalog:",
|
|
||||||
"ts-loader": "catalog:",
|
|
||||||
"ts-node": "catalog:",
|
|
||||||
"tsconfig-paths": "catalog:",
|
|
||||||
"typescript": "catalog:",
|
|
||||||
"typescript-eslint": "catalog:",
|
|
||||||
"vitest": "catalog:"
|
|
||||||
},
|
|
||||||
"jest": {
|
|
||||||
"moduleFileExtensions": [
|
|
||||||
"js",
|
|
||||||
"json",
|
|
||||||
"ts"
|
|
||||||
],
|
|
||||||
"rootDir": "src",
|
|
||||||
"testRegex": ".*\\.spec\\.ts$",
|
|
||||||
"transform": {
|
|
||||||
"^.+\\.(t|j)s$": "ts-jest"
|
|
||||||
},
|
|
||||||
"moduleNameMapper": {
|
|
||||||
"^src/(.*)$": "<rootDir>/$1"
|
|
||||||
},
|
|
||||||
"collectCoverageFrom": [
|
|
||||||
"**/*.(t|j)s"
|
|
||||||
],
|
|
||||||
"coverageDirectory": "../coverage",
|
|
||||||
"testEnvironment": "node"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
import {
|
|
||||||
LoggerRepository,
|
|
||||||
LoggingInterceptor,
|
|
||||||
OtelModule,
|
|
||||||
shutdownOtel,
|
|
||||||
WideContextRepository,
|
|
||||||
} from '@common/server/otel';
|
|
||||||
import { Module, type OnApplicationShutdown, Provider } from '@nestjs/common';
|
|
||||||
import { APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
|
|
||||||
import { JwtModule } from '@nestjs/jwt';
|
|
||||||
import { ZodSerializerInterceptor, ZodValidationPipe } from 'nestjs-zod';
|
|
||||||
import { AppController } from './controllers/app.controller';
|
|
||||||
import { env } from './env';
|
|
||||||
import { AuthGuard } from './middleware/auth.guard';
|
|
||||||
import { ResticInterceptor } from './middleware/restic.interceptor';
|
|
||||||
import { StorageRepository } from './repositories/storage.repository';
|
|
||||||
import { AppService } from './services/app.service';
|
|
||||||
import { AuthService } from './services/auth.service';
|
|
||||||
|
|
||||||
export const imports = [
|
|
||||||
JwtModule.register({
|
|
||||||
global: true,
|
|
||||||
publicKey: env.JWT_PUBLIC_KEY,
|
|
||||||
verifyOptions: { algorithms: ['ES256'] },
|
|
||||||
}),
|
|
||||||
];
|
|
||||||
|
|
||||||
export const controllers = [AppController];
|
|
||||||
|
|
||||||
export const providers: Provider[] = [
|
|
||||||
WideContextRepository,
|
|
||||||
LoggerRepository,
|
|
||||||
StorageRepository,
|
|
||||||
AuthService,
|
|
||||||
AppService,
|
|
||||||
{ provide: APP_GUARD, useClass: AuthGuard },
|
|
||||||
{ provide: APP_INTERCEPTOR, useClass: ResticInterceptor },
|
|
||||||
{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor },
|
|
||||||
{ provide: APP_INTERCEPTOR, useClass: ZodSerializerInterceptor },
|
|
||||||
{ provide: APP_PIPE, useClass: ZodValidationPipe },
|
|
||||||
];
|
|
||||||
|
|
||||||
@Module({
|
|
||||||
imports: [OtelModule, ...imports],
|
|
||||||
controllers,
|
|
||||||
providers,
|
|
||||||
})
|
|
||||||
export class AppModule implements OnApplicationShutdown {
|
|
||||||
async onApplicationShutdown() {
|
|
||||||
await shutdownOtel();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
import { Traceable } from '@common/server/otel';
|
|
||||||
import {
|
|
||||||
Controller,
|
|
||||||
Delete,
|
|
||||||
Get,
|
|
||||||
Head,
|
|
||||||
Headers,
|
|
||||||
HttpCode,
|
|
||||||
HttpStatus,
|
|
||||||
Param,
|
|
||||||
ParseBoolPipe,
|
|
||||||
Post,
|
|
||||||
Query,
|
|
||||||
Req,
|
|
||||||
Res,
|
|
||||||
} from '@nestjs/common';
|
|
||||||
import { type Request, type Response } from 'express';
|
|
||||||
import { BlobInfoResponseDto } from 'src/dto/app.dto';
|
|
||||||
import { type AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { Auth, AuthRoute } from 'src/middleware/auth.guard';
|
|
||||||
import { ResticRoute } from 'src/middleware/restic.interceptor';
|
|
||||||
import { AppService } from 'src/services/app.service';
|
|
||||||
import { respondWithObject } from 'src/utils/s3';
|
|
||||||
import { BlobParamsDto, BlobWithNameParamsDto } from 'src/validation';
|
|
||||||
|
|
||||||
@Traceable()
|
|
||||||
@Controller()
|
|
||||||
export class AppController {
|
|
||||||
constructor(private readonly service: AppService) {}
|
|
||||||
|
|
||||||
@Post(':path')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async createRepository(@Auth() auth: AuthDto, @Query('create', ParseBoolPipe) isCreate: boolean): Promise<void> {
|
|
||||||
await this.service.createRepository(auth.repository, isCreate);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Delete(':path')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.NOT_IMPLEMENTED)
|
|
||||||
deleteRepository(@Auth() _auth: AuthDto): void {
|
|
||||||
this.service.deleteRepository();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Head(':path/config')
|
|
||||||
@AuthRoute()
|
|
||||||
async checkConfig(@Auth() auth: AuthDto, @Res() res: Response): Promise<void> {
|
|
||||||
const size = await this.service.checkConfig(auth);
|
|
||||||
res.set('Content-Length', String(size)).end();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get(':path/config')
|
|
||||||
@AuthRoute()
|
|
||||||
async getConfig(@Auth() auth: AuthDto, @Req() req: Request, @Res() res: Response): Promise<void> {
|
|
||||||
const config = await this.service.getConfig(auth);
|
|
||||||
respondWithObject(config, req, res);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':path/config')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async saveConfig(@Auth() auth: AuthDto, @Req() req: Request): Promise<void> {
|
|
||||||
await this.service.saveConfig(auth, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Delete(':path/config')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async deleteConfig(@Auth() auth: AuthDto): Promise<void> {
|
|
||||||
await this.service.deleteConfig(auth);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get(':path/:type')
|
|
||||||
@AuthRoute()
|
|
||||||
@ResticRoute()
|
|
||||||
async listBlobs(@Auth() auth: AuthDto, @Param() { type }: BlobParamsDto): Promise<BlobInfoResponseDto[]> {
|
|
||||||
return this.service.listBlobs(auth, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Head(':path/:type/:name')
|
|
||||||
@AuthRoute()
|
|
||||||
async checkBlob(
|
|
||||||
@Auth() auth: AuthDto,
|
|
||||||
@Param() { type, name }: BlobWithNameParamsDto,
|
|
||||||
@Res() res: Response,
|
|
||||||
): Promise<void> {
|
|
||||||
const size = await this.service.checkBlob(auth, type, name);
|
|
||||||
res.set('Content-Length', String(size)).end();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get(':path/:type/:name')
|
|
||||||
@AuthRoute()
|
|
||||||
async getBlob(
|
|
||||||
@Auth() auth: AuthDto,
|
|
||||||
@Param() { type, name }: BlobWithNameParamsDto,
|
|
||||||
@Headers('range') range: string | undefined,
|
|
||||||
@Req() req: Request,
|
|
||||||
@Res() res: Response,
|
|
||||||
): Promise<void> {
|
|
||||||
const blob = await this.service.getBlob(auth, type, name, range);
|
|
||||||
respondWithObject(blob, req, res);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':path/:type/:name')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async saveBlob(
|
|
||||||
@Auth() auth: AuthDto,
|
|
||||||
@Param() { type, name }: BlobWithNameParamsDto,
|
|
||||||
@Req() req: Request,
|
|
||||||
): Promise<void> {
|
|
||||||
await this.service.saveBlob(auth, type, name, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Delete(':path/:type/:name')
|
|
||||||
@AuthRoute()
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async deleteBlob(@Auth() auth: AuthDto, @Param() { type, name }: BlobWithNameParamsDto): Promise<void> {
|
|
||||||
await this.service.deleteBlob(auth, type, name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
import { createZodDto } from 'nestjs-zod';
|
|
||||||
import { z } from 'zod';
|
|
||||||
|
|
||||||
const BlobInfoResponseSchema = z
|
|
||||||
.object({
|
|
||||||
name: z.string(),
|
|
||||||
size: z.number(),
|
|
||||||
})
|
|
||||||
.meta({ id: 'BlobInfoResponseDto' });
|
|
||||||
|
|
||||||
export class BlobInfoResponseDto extends createZodDto(BlobInfoResponseSchema) {}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
import { createZodDto } from 'nestjs-zod';
|
|
||||||
import { z } from 'zod';
|
|
||||||
|
|
||||||
const AuthSchema = z
|
|
||||||
.object({
|
|
||||||
user: z.uuid(),
|
|
||||||
repository: z.uuid(),
|
|
||||||
writeOnce: z.boolean(),
|
|
||||||
})
|
|
||||||
.meta({ id: 'AuthDto' });
|
|
||||||
|
|
||||||
export class AuthDto extends createZodDto(AuthSchema) {}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
export enum ContentType {
|
|
||||||
Binary = 'application/octet-stream',
|
|
||||||
ResticV2 = 'application/vnd.x.restic.rest.v2',
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum MetadataKey {
|
|
||||||
Auth = 'AUTH',
|
|
||||||
Restic = 'RESTIC_V2',
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum BlobType {
|
|
||||||
Data = 'data',
|
|
||||||
Index = 'index',
|
|
||||||
Keys = 'keys',
|
|
||||||
Locks = 'locks',
|
|
||||||
Snapshots = 'snapshots',
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
import { z } from 'zod';
|
|
||||||
|
|
||||||
const schema = z.object({
|
|
||||||
NODE_ENV: z.enum(['development', 'production', 'test', 'provision']).default('development'),
|
|
||||||
|
|
||||||
RESTIC_API_PORT: z.coerce.number().min(1000),
|
|
||||||
|
|
||||||
JWT_PUBLIC_KEY: z.string(),
|
|
||||||
|
|
||||||
S3_ACCESS_KEY_ID: z.string(),
|
|
||||||
S3_SECRET_ACCESS_KEY: z.string(),
|
|
||||||
S3_REGION: z.string(),
|
|
||||||
S3_ENDPOINT: z.string(),
|
|
||||||
S3_FORCE_PATH_STYLE: z.coerce.boolean().default(false),
|
|
||||||
|
|
||||||
OTEL_DEBUG: z.coerce.boolean().default(false),
|
|
||||||
OTEL_SAMPLE_RATE: z.number().min(0).max(1).default(1),
|
|
||||||
OTEL_METRICS_EXPORT_INTERVAL: z.number().default(10_000),
|
|
||||||
OTEL_METRICS: z.string().default('http://localhost:8428/opentelemetry/v1/metrics'),
|
|
||||||
OTEL_TRACING: z.string().default('http://localhost:10428/insert/opentelemetry/v1/traces'),
|
|
||||||
OTEL_LOGGING: z.string().default('http://localhost:9428/insert/opentelemetry/v1/logs'),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const env = schema.parse(process.env);
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import { BadRequestException, InternalServerErrorException } from '@nestjs/common';
|
|
||||||
|
|
||||||
export class S3Error extends InternalServerErrorException {
|
|
||||||
constructor() {
|
|
||||||
super('An error occurred with the storage server');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ChecksumMismatchError extends BadRequestException {
|
|
||||||
constructor() {
|
|
||||||
super('Content hash does not match blob name');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import '@common/server/otel';
|
|
||||||
|
|
||||||
import { NestFactory } from '@nestjs/core';
|
|
||||||
import { AppModule } from './app.module';
|
|
||||||
import { env } from './env';
|
|
||||||
|
|
||||||
async function bootstrap() {
|
|
||||||
const app = await NestFactory.create(AppModule);
|
|
||||||
app.enableShutdownHooks();
|
|
||||||
await app.listen(env.RESTIC_API_PORT);
|
|
||||||
}
|
|
||||||
|
|
||||||
void bootstrap();
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
import {
|
|
||||||
BadRequestException,
|
|
||||||
CanActivate,
|
|
||||||
ExecutionContext,
|
|
||||||
Injectable,
|
|
||||||
Scope,
|
|
||||||
SetMetadata,
|
|
||||||
applyDecorators,
|
|
||||||
createParamDecorator,
|
|
||||||
} from '@nestjs/common';
|
|
||||||
import { Reflector } from '@nestjs/core';
|
|
||||||
import { Request } from 'express';
|
|
||||||
import { type AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { MetadataKey } from 'src/enum';
|
|
||||||
import { AuthService } from 'src/services/auth.service';
|
|
||||||
|
|
||||||
export const AuthRoute = (options = {}): MethodDecorator => {
|
|
||||||
return applyDecorators(SetMetadata(MetadataKey.Auth, options));
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface AuthRequest extends Request {
|
|
||||||
auth?: AuthDto;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface AuthenticatedRequest extends Request {
|
|
||||||
auth: AuthDto;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const Auth = createParamDecorator((_, context: ExecutionContext): AuthDto => {
|
|
||||||
return context.switchToHttp().getRequest<AuthenticatedRequest>().auth;
|
|
||||||
});
|
|
||||||
|
|
||||||
@Injectable({ scope: Scope.REQUEST })
|
|
||||||
export class AuthGuard implements CanActivate {
|
|
||||||
constructor(
|
|
||||||
private reflector: Reflector,
|
|
||||||
private service: AuthService,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
|
||||||
const targets = [context.getHandler()];
|
|
||||||
const options = this.reflector.getAllAndOverride<{ _emptyObject: never } | undefined>(MetadataKey.Auth, targets);
|
|
||||||
if (!options) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const request = context.switchToHttp().getRequest<AuthRequest>();
|
|
||||||
request.auth = await this.service.authenticate(request.headers);
|
|
||||||
|
|
||||||
const path = request.params.path;
|
|
||||||
if (path && path !== request.auth.repository) {
|
|
||||||
throw new BadRequestException('Repository mismatch');
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
import {
|
|
||||||
CallHandler,
|
|
||||||
ExecutionContext,
|
|
||||||
Injectable,
|
|
||||||
NestInterceptor,
|
|
||||||
NotImplementedException,
|
|
||||||
SetMetadata,
|
|
||||||
applyDecorators,
|
|
||||||
} from '@nestjs/common';
|
|
||||||
import { Reflector } from '@nestjs/core';
|
|
||||||
import { Request, Response } from 'express';
|
|
||||||
import { Observable, map } from 'rxjs';
|
|
||||||
import { ContentType, MetadataKey } from 'src/enum';
|
|
||||||
|
|
||||||
export const ResticRoute = (): MethodDecorator => {
|
|
||||||
return applyDecorators(SetMetadata(MetadataKey.Restic, true));
|
|
||||||
};
|
|
||||||
|
|
||||||
@Injectable()
|
|
||||||
export class ResticInterceptor implements NestInterceptor {
|
|
||||||
constructor(private reflector: Reflector) {}
|
|
||||||
|
|
||||||
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
|
||||||
const isResticV2 = this.reflector.get<boolean>(MetadataKey.Restic, context.getHandler());
|
|
||||||
if (!isResticV2) {
|
|
||||||
return next.handle();
|
|
||||||
}
|
|
||||||
|
|
||||||
const request = context.switchToHttp().getRequest<Request>();
|
|
||||||
const response = context.switchToHttp().getResponse<Response>();
|
|
||||||
|
|
||||||
if (request.headers.accept !== ContentType.ResticV2) {
|
|
||||||
throw new NotImplementedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
return next.handle().pipe(
|
|
||||||
map((data) => {
|
|
||||||
response.setHeader('Content-Type', ContentType.ResticV2);
|
|
||||||
response.end(JSON.stringify(data));
|
|
||||||
return void 0;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
import {
|
|
||||||
CreateBucketCommand,
|
|
||||||
DeleteObjectCommand,
|
|
||||||
GetObjectCommand,
|
|
||||||
HeadBucketCommand,
|
|
||||||
HeadObjectCommand,
|
|
||||||
ListObjectsV2Command,
|
|
||||||
NotFound,
|
|
||||||
PutObjectCommandInput,
|
|
||||||
S3Client,
|
|
||||||
} from '@aws-sdk/client-s3';
|
|
||||||
import { Upload } from '@aws-sdk/lib-storage';
|
|
||||||
import { Traceable } from '@common/server/otel';
|
|
||||||
import { Injectable } from '@nestjs/common';
|
|
||||||
import { createHash } from 'node:crypto';
|
|
||||||
import { Readable, Transform } from 'node:stream';
|
|
||||||
import { env } from 'src/env';
|
|
||||||
import { ChecksumMismatchError } from 'src/errors';
|
|
||||||
|
|
||||||
@Traceable()
|
|
||||||
@Injectable()
|
|
||||||
export class StorageRepository {
|
|
||||||
private client: S3Client;
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
this.client = new S3Client({
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: env.S3_ACCESS_KEY_ID,
|
|
||||||
secretAccessKey: env.S3_SECRET_ACCESS_KEY,
|
|
||||||
},
|
|
||||||
region: env.S3_REGION,
|
|
||||||
endpoint: env.S3_ENDPOINT,
|
|
||||||
forcePathStyle: env.S3_FORCE_PATH_STYLE,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async checkBucket(Bucket: string): Promise<boolean> {
|
|
||||||
try {
|
|
||||||
await this.client.send(new HeadBucketCommand({ Bucket }));
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof NotFound) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
createBucket(Bucket: string) {
|
|
||||||
return this.client.send(
|
|
||||||
new CreateBucketCommand({
|
|
||||||
Bucket,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async putObject(Bucket: string, Key: string, Body: Readable, writeOnce: boolean, sha256Hex?: string) {
|
|
||||||
const hash = sha256Hex ? createHash('sha256') : undefined;
|
|
||||||
|
|
||||||
const params: PutObjectCommandInput = {
|
|
||||||
Bucket,
|
|
||||||
Key,
|
|
||||||
Body: hash
|
|
||||||
? Body.pipe(
|
|
||||||
new Transform({
|
|
||||||
transform(chunk, _, callback) {
|
|
||||||
hash.update(chunk);
|
|
||||||
callback(null, chunk);
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
: Body,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (writeOnce) {
|
|
||||||
params.IfNoneMatch = '*';
|
|
||||||
}
|
|
||||||
|
|
||||||
const upload = new Upload({
|
|
||||||
client: this.client,
|
|
||||||
params,
|
|
||||||
});
|
|
||||||
|
|
||||||
await upload.done();
|
|
||||||
|
|
||||||
if (hash && hash.digest('hex') !== sha256Hex) {
|
|
||||||
await this.deleteObject(Bucket, Key);
|
|
||||||
throw new ChecksumMismatchError();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
headObject(Bucket: string, Key: string) {
|
|
||||||
return this.client.send(
|
|
||||||
new HeadObjectCommand({
|
|
||||||
Bucket,
|
|
||||||
Key,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async listObjects(Bucket: string, Prefix: string) {
|
|
||||||
return await this.client.send(
|
|
||||||
new ListObjectsV2Command({
|
|
||||||
Bucket,
|
|
||||||
Prefix,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async getObject(Bucket: string, Key: string, Range?: string) {
|
|
||||||
return await this.client.send(
|
|
||||||
new GetObjectCommand({
|
|
||||||
Bucket,
|
|
||||||
Key,
|
|
||||||
Range,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
deleteObject(Bucket: string, Key: string) {
|
|
||||||
return this.client.send(
|
|
||||||
new DeleteObjectCommand({
|
|
||||||
Bucket,
|
|
||||||
Key,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,404 +0,0 @@
|
|||||||
import { S3ServiceException } from '@aws-sdk/client-s3';
|
|
||||||
import { Readable } from 'node:stream';
|
|
||||||
import { text } from 'node:stream/consumers';
|
|
||||||
import { AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { BlobType } from 'src/enum';
|
|
||||||
import { type Mocks, newMocks } from '../../test/mocks';
|
|
||||||
import { AppService } from './app.service';
|
|
||||||
|
|
||||||
const mockAuth = (writeOnce = false): AuthDto => ({
|
|
||||||
user: 'user',
|
|
||||||
repository: 'repository',
|
|
||||||
writeOnce,
|
|
||||||
});
|
|
||||||
|
|
||||||
describe(AppService.name, () => {
|
|
||||||
let mocks: Mocks;
|
|
||||||
let sut: AppService;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
mocks = newMocks();
|
|
||||||
sut = new AppService(mocks.storage as never, mocks.metricService, mocks.wideContext);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should exist', () => {
|
|
||||||
expect(sut).toBeDefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('createRepository', () => {
|
|
||||||
it('should fail if isCreate is false', async () => {
|
|
||||||
await expect(sut.createRepository('repository', false)).rejects.toThrow();
|
|
||||||
expect(mocks.storage.checkBucket).toHaveBeenCalledTimes(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail if bucket exists', async () => {
|
|
||||||
mocks.storage.checkBucket.mockResolvedValue(true);
|
|
||||||
await expect(sut.createRepository('repository', true)).rejects.toThrow();
|
|
||||||
expect(mocks.storage.checkBucket).toHaveBeenCalled();
|
|
||||||
expect(mocks.storage.createBucket).toHaveBeenCalledTimes(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should succeed if bucket does not exist', async () => {
|
|
||||||
mocks.storage.checkBucket.mockResolvedValue(false);
|
|
||||||
await sut.createRepository('repository', true);
|
|
||||||
expect(mocks.storage.checkBucket).toHaveBeenCalled();
|
|
||||||
expect(mocks.storage.createBucket).toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail if S3 command throws', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.checkBucket.mockRejectedValueOnce(S3Error);
|
|
||||||
await expect(sut.createRepository('repository', true)).rejects.toThrow();
|
|
||||||
expect(mocks.storage.checkBucket).toHaveBeenCalled();
|
|
||||||
expect(mocks.storage.createBucket).toHaveBeenCalledTimes(0);
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should fail if S3 command throws', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.createBucket.mockRejectedValueOnce(S3Error);
|
|
||||||
await expect(sut.createRepository('repository', true)).rejects.toThrow();
|
|
||||||
expect(mocks.storage.checkBucket).toHaveBeenCalled();
|
|
||||||
expect(mocks.storage.createBucket).toHaveBeenCalled();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('deleteRepository', () => {
|
|
||||||
it('should do nothing', () => {
|
|
||||||
sut.deleteRepository();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('checkConfig', () => {
|
|
||||||
it('should return content length', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 123, $metadata: void 0 as never });
|
|
||||||
const result = await sut.checkConfig(mockAuth());
|
|
||||||
expect(result).toBe(123);
|
|
||||||
expect(mocks.storage.headObject).toHaveBeenCalledWith('repository', 'config');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return 0 if content length is undefined', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ $metadata: void 0 as never });
|
|
||||||
const result = await sut.checkConfig(mockAuth());
|
|
||||||
expect(result).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if headObject fails', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.headObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.checkConfig(mockAuth())).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('getConfig', () => {
|
|
||||||
it('should return the stream', async () => {
|
|
||||||
const result = await sut.getConfig(mockAuth());
|
|
||||||
expect(result).toEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
object: expect.objectContaining({
|
|
||||||
ContentLength: expect.any(Number),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(mocks.storage.getObject).toHaveBeenCalledWith('repository', 'config');
|
|
||||||
|
|
||||||
const data = await text(result.stream()!);
|
|
||||||
expect(data).toHaveLength(result.object.ContentLength!);
|
|
||||||
expect(sut.blobsDownloadedBytes.add).toHaveBeenCalledWith(
|
|
||||||
result.object.ContentLength,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsDownloadedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
expect(sut.blobsRequestedBytes.add).toHaveBeenCalledWith(
|
|
||||||
result.object.ContentLength,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsRequestedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if getObject fails', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.getObject.mockImplementation(() => new Promise((_, reject) => reject(S3Error)));
|
|
||||||
await expect(sut.getConfig(mockAuth())).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('saveConfig', () => {
|
|
||||||
it('should save config', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
await sut.saveConfig(mockAuth(), body as never);
|
|
||||||
expect(mocks.storage.putObject).toHaveBeenCalledWith('repository', 'config', expect.anything(), false);
|
|
||||||
|
|
||||||
expect(sut.blobsUploadedBytes.add).toHaveBeenCalledWith(
|
|
||||||
4,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsUploadedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should pass writeOnce flag', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
await sut.saveConfig(mockAuth(true), body as never);
|
|
||||||
expect(mocks.storage.putObject).toHaveBeenCalledWith('repository', 'config', expect.anything(), true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw on 412 error', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
const error = new S3ServiceException({
|
|
||||||
name: 'PreconditionFailed',
|
|
||||||
$fault: 'client',
|
|
||||||
$metadata: { httpStatusCode: 412 },
|
|
||||||
});
|
|
||||||
mocks.storage.putObject.mockRejectedValue(error);
|
|
||||||
await expect(sut.saveConfig(mockAuth(true), body as never)).rejects.toThrow('Config already exists');
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw on other errors', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.putObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.saveConfig(mockAuth(), body as never)).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('deleteConfig', () => {
|
|
||||||
it('should delete config', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 500, $metadata: void 0 as never });
|
|
||||||
mocks.storage.deleteObject.mockResolvedValue(void 0 as never);
|
|
||||||
await sut.deleteConfig(mockAuth());
|
|
||||||
expect(mocks.storage.headObject).toHaveBeenCalledWith('repository', 'config');
|
|
||||||
expect(mocks.storage.deleteObject).toHaveBeenCalledWith('repository', 'config');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw when writeOnce and not locks', async () => {
|
|
||||||
await expect(sut.deleteConfig(mockAuth(true))).rejects.toThrow();
|
|
||||||
expect(mocks.storage.deleteObject).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if deleteObject fails', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 500, $metadata: void 0 as never });
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.deleteObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.deleteConfig(mockAuth())).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('listBlobs', () => {
|
|
||||||
it('should return mapped blobs', async () => {
|
|
||||||
mocks.storage.listObjects.mockResolvedValue({
|
|
||||||
Contents: [
|
|
||||||
{ Key: 'data/abc123', Size: 100 },
|
|
||||||
{ Key: 'data/def456', Size: 200 },
|
|
||||||
],
|
|
||||||
KeyCount: 2,
|
|
||||||
$metadata: void 0 as never,
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await sut.listBlobs(mockAuth(), BlobType.Data);
|
|
||||||
expect(result).toEqual([
|
|
||||||
{ name: 'abc123', size: 100 },
|
|
||||||
{ name: 'def456', size: 200 },
|
|
||||||
]);
|
|
||||||
expect(mocks.storage.listObjects).toHaveBeenCalledWith('repository', 'data/');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return empty array when KeyCount is 0', async () => {
|
|
||||||
mocks.storage.listObjects.mockResolvedValue({ KeyCount: 0, $metadata: void 0 as never });
|
|
||||||
const result = await sut.listBlobs(mockAuth(), BlobType.Data);
|
|
||||||
expect(result).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if Contents is undefined', async () => {
|
|
||||||
mocks.storage.listObjects.mockResolvedValue({ KeyCount: 1, $metadata: void 0 as never });
|
|
||||||
await expect(sut.listBlobs(mockAuth(), BlobType.Data)).rejects.toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if Key or Size is missing', async () => {
|
|
||||||
const Contents = [{ Key: 'data/abc123' }];
|
|
||||||
mocks.storage.listObjects.mockResolvedValue({
|
|
||||||
Contents,
|
|
||||||
KeyCount: 1,
|
|
||||||
$metadata: void 0 as never,
|
|
||||||
});
|
|
||||||
|
|
||||||
await expect(sut.listBlobs(mockAuth(), BlobType.Data)).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.addContext).toHaveBeenCalledWith('contents', Contents);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if listObjects fails', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.listObjects.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.listBlobs(mockAuth(), BlobType.Data)).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('checkBlob', () => {
|
|
||||||
it('should return content length', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 456, $metadata: void 0 as never });
|
|
||||||
const result = await sut.checkBlob(mockAuth(), BlobType.Data, 'abc123');
|
|
||||||
expect(result).toBe(456);
|
|
||||||
expect(mocks.storage.headObject).toHaveBeenCalledWith('repository', 'data/abc123');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return 0 if content length is undefined', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ $metadata: void 0 as never });
|
|
||||||
const result = await sut.checkBlob(mockAuth(), BlobType.Data, 'abc123');
|
|
||||||
expect(result).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if headObject fails', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.headObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.checkBlob(mockAuth(), BlobType.Data, 'abc123')).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('getBlob', () => {
|
|
||||||
it('should return the object', async () => {
|
|
||||||
const result = await sut.getBlob(mockAuth(), BlobType.Data, 'abc123');
|
|
||||||
expect(result).toEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
object: expect.objectContaining({
|
|
||||||
ContentLength: expect.any(Number),
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(mocks.storage.getObject).toHaveBeenCalledWith('repository', 'data/abc123', undefined);
|
|
||||||
|
|
||||||
const data = await text(result.stream()!);
|
|
||||||
expect(data).toHaveLength(result.object.ContentLength!);
|
|
||||||
expect(sut.blobsDownloadedBytes.add).toHaveBeenCalledWith(
|
|
||||||
result.object.ContentLength,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsDownloadedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
expect(sut.blobsRequestedBytes.add).toHaveBeenCalledWith(
|
|
||||||
result.object.ContentLength,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsRequestedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should pass range to getObjectStream', async () => {
|
|
||||||
await sut.getBlob(mockAuth(), BlobType.Data, 'abc123', 'bytes=0-100');
|
|
||||||
expect(mocks.storage.getObject).toHaveBeenCalledWith('repository', 'data/abc123', 'bytes=0-100');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if getObject fails', async () => {
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.getObject.mockImplementation(() => new Promise((_, reject) => reject(S3Error)));
|
|
||||||
await expect(sut.getBlob(mockAuth(), BlobType.Data, 'abc123')).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('saveBlob', () => {
|
|
||||||
it('should save blob', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
await sut.saveBlob(mockAuth(), BlobType.Data, 'abc123', body as never);
|
|
||||||
expect(mocks.storage.putObject).toHaveBeenCalledWith(
|
|
||||||
'repository',
|
|
||||||
'data/abc123',
|
|
||||||
expect.anything(),
|
|
||||||
true,
|
|
||||||
'abc123',
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(sut.blobsUploadedBytes.add).toHaveBeenCalledWith(
|
|
||||||
4,
|
|
||||||
expect.objectContaining({
|
|
||||||
customerId: 'user',
|
|
||||||
repositoryId: 'repository',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sut.blobsUploadedBytes.add).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should pass writeOnce flag', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
await sut.saveBlob(mockAuth(true), BlobType.Data, 'abc123', body as never);
|
|
||||||
expect(mocks.storage.putObject).toHaveBeenCalledWith(
|
|
||||||
'repository',
|
|
||||||
'data/abc123',
|
|
||||||
expect.anything(),
|
|
||||||
true,
|
|
||||||
'abc123',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw ConflictException on 412 error', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
const error = new S3ServiceException({
|
|
||||||
name: 'PreconditionFailed',
|
|
||||||
$fault: 'client',
|
|
||||||
$metadata: { httpStatusCode: 412 },
|
|
||||||
});
|
|
||||||
mocks.storage.putObject.mockRejectedValue(error);
|
|
||||||
await expect(sut.saveBlob(mockAuth(true), BlobType.Data, 'abc123', body as never)).rejects.toThrow(
|
|
||||||
'Blob already exists',
|
|
||||||
);
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw on other errors', async () => {
|
|
||||||
const body = Readable.from('body');
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.putObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.saveBlob(mockAuth(), BlobType.Data, 'abc123', body as never)).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('deleteBlob', () => {
|
|
||||||
it('should delete blob', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 500, $metadata: void 0 as never });
|
|
||||||
mocks.storage.deleteObject.mockResolvedValue(void 0 as never);
|
|
||||||
await sut.deleteBlob(mockAuth(), BlobType.Data, 'abc123');
|
|
||||||
expect(mocks.storage.headObject).toHaveBeenCalledWith('repository', 'data/abc123');
|
|
||||||
expect(mocks.storage.deleteObject).toHaveBeenCalledWith('repository', 'data/abc123');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should allow delete of locks with writeOnce', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 500, $metadata: void 0 as never });
|
|
||||||
mocks.storage.deleteObject.mockResolvedValue(void 0 as never);
|
|
||||||
await sut.deleteBlob(mockAuth(true), BlobType.Locks, 'abc123');
|
|
||||||
expect(mocks.storage.deleteObject).toHaveBeenCalledWith('repository', 'locks/abc123');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw when writeOnce and not locks', async () => {
|
|
||||||
await expect(sut.deleteBlob(mockAuth(true), BlobType.Data, 'abc123')).rejects.toThrow();
|
|
||||||
expect(mocks.storage.deleteObject).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if deleteObject fails', async () => {
|
|
||||||
mocks.storage.headObject.mockResolvedValue({ ContentLength: 500, $metadata: void 0 as never });
|
|
||||||
const S3Error = Symbol('S3Error');
|
|
||||||
mocks.storage.deleteObject.mockRejectedValue(S3Error);
|
|
||||||
await expect(sut.deleteBlob(mockAuth(), BlobType.Data, 'abc123')).rejects.toThrow();
|
|
||||||
expect(mocks.wideContext.setErrorCause).toHaveBeenCalledWith(S3Error);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,254 +0,0 @@
|
|||||||
import { S3ServiceException } from '@aws-sdk/client-s3';
|
|
||||||
import { MetricService, Traceable, WideContextRepository } from '@common/server/otel';
|
|
||||||
import {
|
|
||||||
BadRequestException,
|
|
||||||
ConflictException,
|
|
||||||
ForbiddenException,
|
|
||||||
Injectable,
|
|
||||||
NotFoundException,
|
|
||||||
} from '@nestjs/common';
|
|
||||||
import { Counter, UpDownCounter } from '@opentelemetry/api';
|
|
||||||
import { Readable } from 'node:stream';
|
|
||||||
import { BlobInfoResponseDto } from 'src/dto/app.dto';
|
|
||||||
import { AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { BlobType } from 'src/enum';
|
|
||||||
import { ChecksumMismatchError, S3Error } from 'src/errors';
|
|
||||||
import { StorageRepository } from 'src/repositories/storage.repository';
|
|
||||||
import { attachMeterToStream, contextFromAuth } from 'src/utils/meters';
|
|
||||||
import { attachMeterToS3GetObject, S3RemoteObject } from 'src/utils/s3';
|
|
||||||
|
|
||||||
@Traceable()
|
|
||||||
@Injectable()
|
|
||||||
export class AppService {
|
|
||||||
blobsRequestedBytes: Counter;
|
|
||||||
blobsDownloadedBytes: Counter;
|
|
||||||
blobsUploadedBytes: Counter;
|
|
||||||
blobsStoredBytes: UpDownCounter;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private readonly storage: StorageRepository,
|
|
||||||
private readonly metricService: MetricService,
|
|
||||||
private readonly wideContext: WideContextRepository,
|
|
||||||
) {
|
|
||||||
this.blobsRequestedBytes = this.metricService.getCounter('blobs.requested_bytes', {
|
|
||||||
description: 'Total no. of blob bytes requested for download',
|
|
||||||
});
|
|
||||||
|
|
||||||
this.blobsDownloadedBytes = this.metricService.getCounter('blobs.downloaded_bytes', {
|
|
||||||
description: 'Total no. of blob bytes download',
|
|
||||||
});
|
|
||||||
|
|
||||||
this.blobsUploadedBytes = this.metricService.getCounter('blobs.uploaded_bytes', {
|
|
||||||
description: 'Total no. of blob bytes uploaded',
|
|
||||||
});
|
|
||||||
|
|
||||||
this.blobsStoredBytes = this.metricService.getUpDownCounter('blobs.stored_bytes', {
|
|
||||||
description: 'Total no. of blob bytes stored',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async createRepository(repository: string, isCreate: boolean): Promise<void> {
|
|
||||||
if (!isCreate) {
|
|
||||||
throw new BadRequestException('isCreate must be true when creating repository');
|
|
||||||
}
|
|
||||||
|
|
||||||
let exists: boolean;
|
|
||||||
try {
|
|
||||||
exists = await this.storage.checkBucket(repository);
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (exists) {
|
|
||||||
throw new ConflictException('Repository already exists');
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.storage.createBucket(repository);
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
deleteRepository(): void {}
|
|
||||||
|
|
||||||
async checkConfig(auth: AuthDto): Promise<number> {
|
|
||||||
try {
|
|
||||||
const { ContentLength } = await this.storage.headObject(auth.repository, 'config');
|
|
||||||
return ContentLength || 0;
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new NotFoundException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async getConfig(auth: AuthDto): Promise<S3RemoteObject> {
|
|
||||||
try {
|
|
||||||
return attachMeterToS3GetObject(
|
|
||||||
auth,
|
|
||||||
await this.storage.getObject(auth.repository, 'config'),
|
|
||||||
this.blobsRequestedBytes,
|
|
||||||
this.blobsDownloadedBytes,
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async saveConfig(auth: AuthDto, body: Readable): Promise<void> {
|
|
||||||
try {
|
|
||||||
let stored = 0;
|
|
||||||
|
|
||||||
await this.storage.putObject(
|
|
||||||
auth.repository,
|
|
||||||
'config',
|
|
||||||
attachMeterToStream(body, this.blobsUploadedBytes, (bytes) => (stored += bytes), contextFromAuth(auth)),
|
|
||||||
auth.writeOnce,
|
|
||||||
);
|
|
||||||
|
|
||||||
this.blobsStoredBytes.add(stored, contextFromAuth(auth));
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
|
|
||||||
if (error instanceof S3ServiceException && error.$metadata.httpStatusCode === 412) {
|
|
||||||
throw new ForbiddenException('Config already exists');
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async deleteConfig(auth: AuthDto): Promise<void> {
|
|
||||||
if (auth.writeOnce) {
|
|
||||||
throw new ForbiddenException('Not permitted to write to WORM repository');
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const { ContentLength } = await this.storage.headObject(auth.repository, 'config');
|
|
||||||
|
|
||||||
if (!ContentLength) {
|
|
||||||
throw 'Missing ContentLength from headObject output';
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.storage.deleteObject(auth.repository, 'config');
|
|
||||||
|
|
||||||
this.blobsStoredBytes.add(-ContentLength, contextFromAuth(auth));
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async listBlobs(auth: AuthDto, type: BlobType): Promise<BlobInfoResponseDto[]> {
|
|
||||||
try {
|
|
||||||
const suffix = `${type}/`;
|
|
||||||
const { Contents, KeyCount } = await this.storage.listObjects(auth.repository, suffix);
|
|
||||||
|
|
||||||
if (KeyCount === 0) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Contents) {
|
|
||||||
this.wideContext.setErrorCause('Contents missing from ListObjects response');
|
|
||||||
throw void 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Contents.some(({ Key, Size }) => !Key || !Size)) {
|
|
||||||
this.wideContext.setErrorCause('Contents are malformed from ListObjects response');
|
|
||||||
this.wideContext.addContext('contents', Contents);
|
|
||||||
throw void 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Contents!.map(({ Key, Size }) => ({
|
|
||||||
name: Key!.slice(suffix.length),
|
|
||||||
size: Size!,
|
|
||||||
}));
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async checkBlob(auth: AuthDto, type: BlobType, name: string): Promise<number> {
|
|
||||||
try {
|
|
||||||
const { ContentLength } = await this.storage.headObject(auth.repository, `${type}/${name}`);
|
|
||||||
return ContentLength || 0;
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new NotFoundException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async getBlob(auth: AuthDto, type: BlobType, name: string, range?: string): Promise<S3RemoteObject> {
|
|
||||||
try {
|
|
||||||
return attachMeterToS3GetObject(
|
|
||||||
auth,
|
|
||||||
await this.storage.getObject(auth.repository, `${type}/${name}`, range),
|
|
||||||
this.blobsRequestedBytes,
|
|
||||||
this.blobsDownloadedBytes,
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async saveBlob(auth: AuthDto, type: BlobType, name: string, body: Readable): Promise<void> {
|
|
||||||
try {
|
|
||||||
let stored = 0;
|
|
||||||
|
|
||||||
await this.storage.putObject(
|
|
||||||
auth.repository,
|
|
||||||
`${type}/${name}`,
|
|
||||||
attachMeterToStream(body, this.blobsUploadedBytes, (bytes) => (stored += bytes), contextFromAuth(auth)),
|
|
||||||
true,
|
|
||||||
name,
|
|
||||||
);
|
|
||||||
|
|
||||||
this.blobsStoredBytes.add(stored, contextFromAuth(auth));
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
|
|
||||||
if (error instanceof ChecksumMismatchError) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error instanceof S3ServiceException) {
|
|
||||||
if (error.$metadata.httpStatusCode === 412) {
|
|
||||||
throw new ForbiddenException('Blob already exists');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error.name === 'XAmzContentChecksumMismatch') {
|
|
||||||
throw new BadRequestException('Content hash does not match blob name');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async deleteBlob(auth: AuthDto, type: BlobType, name: string): Promise<void> {
|
|
||||||
if (auth.writeOnce && type !== BlobType.Locks) {
|
|
||||||
throw new ForbiddenException('Not permitted to write to WORM repository');
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const { ContentLength } = await this.storage.headObject(auth.repository, `${type}/${name}`);
|
|
||||||
|
|
||||||
if (!ContentLength) {
|
|
||||||
throw 'Missing ContentLength from headObject output';
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.storage.deleteObject(auth.repository, `${type}/${name}`);
|
|
||||||
|
|
||||||
this.blobsStoredBytes.add(-ContentLength, contextFromAuth(auth));
|
|
||||||
} catch (error) {
|
|
||||||
this.wideContext.setErrorCause(error);
|
|
||||||
|
|
||||||
throw new S3Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
|
||||||
import { newJwtMock, newWideContextMock } from '../../test/mocks';
|
|
||||||
import { AuthService } from './auth.service';
|
|
||||||
|
|
||||||
describe(AuthService.name, () => {
|
|
||||||
let jwt: ReturnType<typeof newJwtMock>;
|
|
||||||
let wideContext: ReturnType<typeof newWideContextMock>;
|
|
||||||
let sut: AuthService;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
jwt = newJwtMock();
|
|
||||||
wideContext = newWideContextMock();
|
|
||||||
sut = new AuthService(jwt as never, wideContext as never);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should exist', () => {
|
|
||||||
expect(sut).toBeDefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('authenticate', () => {
|
|
||||||
it('should throw if Authorization header is missing', async () => {
|
|
||||||
await expect(sut.authenticate({})).rejects.toThrow('Missing Authorization header');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if not Basic auth', async () => {
|
|
||||||
await expect(sut.authenticate({ authorization: 'Bearer token' })).rejects.toThrow('Expected Basic auth');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if token is missing', async () => {
|
|
||||||
const auth = Buffer.from('username').toString('base64');
|
|
||||||
await expect(sut.authenticate({ authorization: `Basic ${auth}` })).rejects.toThrow('Expected Basic auth token');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if JWT verification fails', async () => {
|
|
||||||
jwt.verifyAsync.mockRejectedValue(new Error('invalid'));
|
|
||||||
const auth = Buffer.from('username:token').toString('base64');
|
|
||||||
await expect(sut.authenticate({ authorization: `Basic ${auth}` })).rejects.toThrow('Invalid JWT Token');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should throw if JWT payload is invalid', async () => {
|
|
||||||
jwt.verifyAsync.mockResolvedValue({ invalid: 'payload' });
|
|
||||||
const auth = Buffer.from('username:token').toString('base64');
|
|
||||||
await expect(sut.authenticate({ authorization: `Basic ${auth}` })).rejects.toThrow('Bad Request Exception');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return auth dto on success', async () => {
|
|
||||||
const payload = { user: randomUUID(), repository: randomUUID(), writeOnce: true };
|
|
||||||
jwt.verifyAsync.mockResolvedValue(payload);
|
|
||||||
const auth = Buffer.from('username:token').toString('base64');
|
|
||||||
const result = await sut.authenticate({ authorization: `Basic ${auth}` });
|
|
||||||
expect(result).toEqual(payload);
|
|
||||||
expect(jwt.verifyAsync).toHaveBeenCalledWith('token');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
import { WideContextRepository } from '@common/server/otel';
|
|
||||||
import { BadRequestException, Injectable, UnauthorizedException } from '@nestjs/common';
|
|
||||||
import { JwtService } from '@nestjs/jwt';
|
|
||||||
import { type IncomingHttpHeaders } from 'node:http';
|
|
||||||
import { AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { contextFromAuth } from 'src/utils/meters';
|
|
||||||
|
|
||||||
@Injectable()
|
|
||||||
export class AuthService {
|
|
||||||
constructor(
|
|
||||||
private readonly jwt: JwtService,
|
|
||||||
private readonly wideContext: WideContextRepository,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async authenticate(headers: IncomingHttpHeaders): Promise<AuthDto> {
|
|
||||||
if (!headers.authorization) {
|
|
||||||
throw new UnauthorizedException('Missing Authorization header');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!headers.authorization.startsWith('Basic ')) {
|
|
||||||
throw new UnauthorizedException('Expected Basic auth');
|
|
||||||
}
|
|
||||||
|
|
||||||
const auth = Buffer.from(headers.authorization.split(' ').pop() || '', 'base64').toString();
|
|
||||||
const [_, token] = auth.split(':');
|
|
||||||
|
|
||||||
if (!token) {
|
|
||||||
throw new UnauthorizedException('Expected Basic auth token');
|
|
||||||
}
|
|
||||||
|
|
||||||
let jwt;
|
|
||||||
try {
|
|
||||||
jwt = await this.jwt.verifyAsync(token);
|
|
||||||
} catch {
|
|
||||||
throw new UnauthorizedException('Invalid JWT Token');
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = AuthDto.schema.safeParse(jwt);
|
|
||||||
if (!result.success) {
|
|
||||||
throw new BadRequestException(result.error.issues.map((issue) => issue.message));
|
|
||||||
}
|
|
||||||
|
|
||||||
this.wideContext.assignContext(contextFromAuth(result.data));
|
|
||||||
return result.data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
import { Attributes, Counter } from '@opentelemetry/api';
|
|
||||||
import { PassThrough, Readable } from 'node:stream';
|
|
||||||
import { AuthDto } from 'src/dto/auth.dto';
|
|
||||||
|
|
||||||
export function attachMeterToStream(
|
|
||||||
stream: Readable,
|
|
||||||
meterBytes: Counter,
|
|
||||||
callbackBytes: (bytes: number) => void,
|
|
||||||
context: Attributes,
|
|
||||||
) {
|
|
||||||
const passthrough = new PassThrough({
|
|
||||||
transform(chunk, _, callback) {
|
|
||||||
meterBytes.add(chunk.length, context);
|
|
||||||
callbackBytes(chunk.length);
|
|
||||||
callback(null, chunk);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
stream.pipe(passthrough);
|
|
||||||
|
|
||||||
return passthrough;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function contextFromAuth(auth: AuthDto) {
|
|
||||||
return {
|
|
||||||
customerId: auth.user,
|
|
||||||
repositoryId: auth.repository,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
import { GetObjectCommandOutput } from '@aws-sdk/client-s3';
|
|
||||||
import { HttpStatus } from '@nestjs/common';
|
|
||||||
import { Counter } from '@opentelemetry/api';
|
|
||||||
import { Request, Response } from 'express';
|
|
||||||
import { Readable } from 'node:stream';
|
|
||||||
import { ReadableStream } from 'node:stream/web';
|
|
||||||
import { AuthDto } from 'src/dto/auth.dto';
|
|
||||||
import { ContentType } from '../enum';
|
|
||||||
import { attachMeterToStream, contextFromAuth } from './meters';
|
|
||||||
|
|
||||||
export interface S3RemoteObject {
|
|
||||||
stream(): Readable | undefined;
|
|
||||||
object: GetObjectCommandOutput;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function attachMeterToS3GetObject(
|
|
||||||
auth: AuthDto,
|
|
||||||
object: GetObjectCommandOutput,
|
|
||||||
requestedBytes: Counter,
|
|
||||||
downloadedBytes: Counter,
|
|
||||||
): S3RemoteObject {
|
|
||||||
const context = contextFromAuth(auth);
|
|
||||||
requestedBytes.add(object.ContentLength || 0, context);
|
|
||||||
|
|
||||||
return {
|
|
||||||
object,
|
|
||||||
stream() {
|
|
||||||
const webStream = object.Body?.transformToWebStream();
|
|
||||||
if (webStream) {
|
|
||||||
return attachMeterToStream(
|
|
||||||
Readable.fromWeb(webStream as ReadableStream),
|
|
||||||
downloadedBytes,
|
|
||||||
() => void 0,
|
|
||||||
context,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Process S3 object as web response
|
|
||||||
*
|
|
||||||
* References:
|
|
||||||
* http#ServeContent
|
|
||||||
* https://pkg.go.dev/net/http#ServeContent
|
|
||||||
*/
|
|
||||||
export function respondWithObject({ stream, object }: S3RemoteObject, request: Request, response: Response) {
|
|
||||||
if (request.headers['if-none-match'] === object.ETag) {
|
|
||||||
return response.send(HttpStatus.NOT_MODIFIED);
|
|
||||||
}
|
|
||||||
|
|
||||||
const range = request.headers.range;
|
|
||||||
if (range && range !== 'bytes=0-') {
|
|
||||||
response.status(HttpStatus.PARTIAL_CONTENT);
|
|
||||||
} else {
|
|
||||||
response.status(HttpStatus.OK);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (object.ETag) {
|
|
||||||
response.header('ETag', object.ETag);
|
|
||||||
}
|
|
||||||
|
|
||||||
response.set('Content-Type', object.ContentType ?? ContentType.Binary);
|
|
||||||
|
|
||||||
if (object.ContentRange) {
|
|
||||||
response.set('Content-Range', object.ContentRange);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (object.ContentLength) {
|
|
||||||
response.set('Content-Length', object.ContentLength.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
const readable = stream();
|
|
||||||
if (readable) {
|
|
||||||
readable.pipe(response);
|
|
||||||
} else {
|
|
||||||
return response.send(HttpStatus.INTERNAL_SERVER_ERROR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
import { createZodDto } from 'nestjs-zod';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { BlobType } from './enum';
|
|
||||||
|
|
||||||
const BlobParamsSchema = z.object({ type: z.enum(BlobType) }).meta({ id: 'BlobParamsDto' });
|
|
||||||
|
|
||||||
const BlobWithNameParamsSchema = BlobParamsSchema.extend({
|
|
||||||
name: z.string().regex(/^[a-f0-9]{64}$/),
|
|
||||||
}).meta({ id: 'BlobWithNameParamsDto' });
|
|
||||||
|
|
||||||
export class BlobParamsDto extends createZodDto(BlobParamsSchema) {}
|
|
||||||
export class BlobWithNameParamsDto extends createZodDto(BlobWithNameParamsSchema) {}
|
|
||||||
@@ -1,401 +0,0 @@
|
|||||||
import { MetricService } from '@common/server/otel';
|
|
||||||
import { INestApplication } from '@nestjs/common';
|
|
||||||
import { JwtService } from '@nestjs/jwt';
|
|
||||||
import { Test, TestingModule } from '@nestjs/testing';
|
|
||||||
import { createHash, randomUUID } from 'node:crypto';
|
|
||||||
import request from 'supertest';
|
|
||||||
import { App } from 'supertest/types';
|
|
||||||
import { controllers, imports, providers } from '../src/app.module';
|
|
||||||
import { newMetricServiceMock } from './mocks';
|
|
||||||
|
|
||||||
const makeAuthHeader = (token: string) => 'Basic ' + Buffer.from(`_:${token}`).toString('base64');
|
|
||||||
|
|
||||||
describe('AppController (e2e)', () => {
|
|
||||||
let app: INestApplication<App>;
|
|
||||||
|
|
||||||
let repository: string;
|
|
||||||
let authHeader: string;
|
|
||||||
let wormAuthHeader: string;
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
const moduleFixture: TestingModule = await Test.createTestingModule({
|
|
||||||
imports,
|
|
||||||
controllers,
|
|
||||||
providers: [MetricService, ...providers],
|
|
||||||
})
|
|
||||||
.overrideProvider(MetricService)
|
|
||||||
.useValue(newMetricServiceMock())
|
|
||||||
.compile();
|
|
||||||
|
|
||||||
app = moduleFixture.createNestApplication();
|
|
||||||
await app.init();
|
|
||||||
});
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await app.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
const signer = new JwtService({
|
|
||||||
privateKey: process.env.JWT_PRIVATE_KEY,
|
|
||||||
signOptions: { algorithm: 'ES256' },
|
|
||||||
});
|
|
||||||
|
|
||||||
repository = randomUUID();
|
|
||||||
authHeader = makeAuthHeader(signer.sign({ user: randomUUID(), repository, writeOnce: false }));
|
|
||||||
wormAuthHeader = makeAuthHeader(signer.sign({ user: randomUUID(), repository, writeOnce: true }));
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('POST /:path', () => {
|
|
||||||
it('fails if create is not true', async () => {
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=false`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(400);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('creates the repository', async () => {
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails if repository already exists', async () => {
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(409);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('DELETE /:path', () => {
|
|
||||||
it('returns not implemented', async () => {
|
|
||||||
await request(app.getHttpServer()).delete(`/${repository}`).set('Authorization', authHeader).expect(501);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('HEAD /:path/config', () => {
|
|
||||||
it('returns 404 if config does not exist', async () => {
|
|
||||||
await request(app.getHttpServer()).head(`/${repository}/config`).set('Authorization', authHeader).expect(404);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns content-length if config exists', async () => {
|
|
||||||
const config = 'test-config-data';
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/config`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(config))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.head(`/${repository}/config`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Length', String(config.length));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('GET, POST /:path/config', () => {
|
|
||||||
it('saves and returns config data', async () => {
|
|
||||||
const config = 'test-config-data';
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/config`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(config))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
const response = await request(app.getHttpServer())
|
|
||||||
.get(`/${repository}/config`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Type', 'application/octet-stream');
|
|
||||||
|
|
||||||
expect(response.body.toString()).toBe(config);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails to overwrite config with worm', async () => {
|
|
||||||
const config = 'test-config-data';
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/config`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(config))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/config`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(config))
|
|
||||||
.expect(403);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('GET /:path/:type', () => {
|
|
||||||
it('returns 501 without restic v2 accept header', async () => {
|
|
||||||
await request(app.getHttpServer()).get(`/${repository}/data`).set('Authorization', authHeader).expect(501);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns empty list when no blobs exist', async () => {
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
const response = await request(app.getHttpServer())
|
|
||||||
.get(`/${repository}/data`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Accept', 'application/vnd.x.restic.rest.v2')
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Type', 'application/vnd.x.restic.rest.v2');
|
|
||||||
|
|
||||||
expect(JSON.parse(response.text)).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns list of blobs when they exist', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
const response = await request(app.getHttpServer())
|
|
||||||
.get(`/${repository}/data`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Accept', 'application/vnd.x.restic.rest.v2')
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Type', 'application/vnd.x.restic.rest.v2');
|
|
||||||
|
|
||||||
expect(JSON.parse(response.text)).toEqual([{ name: blobName, size: blobData.length }]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('HEAD /:path/:type/:name', () => {
|
|
||||||
it('returns 404 if blob does not exist', async () => {
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.head(`/${repository}/data/${'a'.repeat(64)}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(404);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns content-length if blob exists', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.head(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Length', String(blobData.length));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('GET, POST /:path/:type/:name', () => {
|
|
||||||
it('saves and returns blob data', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
const response = await request(app.getHttpServer())
|
|
||||||
.get(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200)
|
|
||||||
.expect('Content-Type', 'application/octet-stream');
|
|
||||||
|
|
||||||
expect(response.body.toString()).toBe(blobData);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns partial content with range header', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
const response = await request(app.getHttpServer())
|
|
||||||
.get(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Range', 'bytes=0-3')
|
|
||||||
.expect(206)
|
|
||||||
.expect('Content-Type', 'application/octet-stream');
|
|
||||||
|
|
||||||
expect(response.body.toString()).toBe('test');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails to overwrite blob with worm', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(403);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails to write blob with non-hash for name', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = 'invalid-hash';
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(400);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails to write blob with non-matching hash for name', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = 'a'.repeat(64);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(400);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('DELETE /:path/:type/:name', () => {
|
|
||||||
it('deletes a blob', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.delete(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.head(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', authHeader)
|
|
||||||
.expect(404);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails to delete blob with worm', async () => {
|
|
||||||
const blobData = 'test-blob-data';
|
|
||||||
const blobName = createHash('sha256').update(blobData).digest('hex');
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}?create=true`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.post(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.set('Content-Type', 'application/octet-stream')
|
|
||||||
.send(Buffer.from(blobData))
|
|
||||||
.expect(200);
|
|
||||||
|
|
||||||
await request(app.getHttpServer())
|
|
||||||
.delete(`/${repository}/data/${blobName}`)
|
|
||||||
.set('Authorization', wormAuthHeader)
|
|
||||||
.expect(403);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
{
|
|
||||||
"moduleFileExtensions": ["js", "json", "ts"],
|
|
||||||
"rootDir": ".",
|
|
||||||
"testEnvironment": "node",
|
|
||||||
"testRegex": ".integration-spec.ts$",
|
|
||||||
"transform": {
|
|
||||||
"^.+\\.(t|j)s$": "ts-jest"
|
|
||||||
},
|
|
||||||
"moduleNameMapper": {
|
|
||||||
"^src/(.*)$": "<rootDir>/../src/$1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
import type { LoggerRepository, MetricService, WideContextRepository } from '@common/server/otel';
|
|
||||||
import { Readable } from 'node:stream';
|
|
||||||
import { text } from 'node:stream/consumers';
|
|
||||||
import { StorageRepository } from 'src/repositories/storage.repository';
|
|
||||||
|
|
||||||
export type RepositoryInterface<T extends object> = Pick<T, keyof T>;
|
|
||||||
|
|
||||||
export const newJwtMock = () => ({
|
|
||||||
verifyAsync: jest.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const newWideContextMock = () => ({
|
|
||||||
assignContext: jest.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const newLoggerRepositoryMock = (): jest.Mocked<RepositoryInterface<LoggerRepository>> => {
|
|
||||||
return {
|
|
||||||
debug: jest.fn(),
|
|
||||||
error: jest.fn(),
|
|
||||||
info: jest.fn(),
|
|
||||||
warn: jest.fn(),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const newStorageRepositoryMock = (): jest.Mocked<RepositoryInterface<StorageRepository>> => {
|
|
||||||
return {
|
|
||||||
checkBucket: jest.fn(),
|
|
||||||
createBucket: jest.fn(),
|
|
||||||
deleteObject: jest.fn(),
|
|
||||||
getObject: jest.fn().mockResolvedValue({
|
|
||||||
ContentLength: 1000,
|
|
||||||
Body: {
|
|
||||||
transformToWebStream() {
|
|
||||||
return Readable.toWeb(Readable.from('_'.repeat(1000)));
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
headObject: jest.fn(),
|
|
||||||
listObjects: jest.fn(),
|
|
||||||
putObject: jest.fn().mockImplementation((_1, _2, body: Readable) => text(body)),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export const newWideContextRepositoryMock = (): jest.Mocked<RepositoryInterface<WideContextRepository>> => ({
|
|
||||||
context: {},
|
|
||||||
addContext: jest.fn(),
|
|
||||||
applyContext: jest.fn(),
|
|
||||||
assignContext: jest.fn(),
|
|
||||||
setErrorCause: jest.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const newMetricServiceMock = (): jest.Mocked<RepositoryInterface<MetricService>> => ({
|
|
||||||
getCounter: jest.fn().mockImplementation(() => ({ add: jest.fn() })),
|
|
||||||
getGauge: jest.fn(),
|
|
||||||
getHistogram: jest.fn(),
|
|
||||||
getObservableCounter: jest.fn(),
|
|
||||||
getObservableGauge: jest.fn(),
|
|
||||||
getObservableUpDownCounter: jest.fn(),
|
|
||||||
getUpDownCounter: jest.fn().mockImplementation(() => ({ add: jest.fn() })),
|
|
||||||
});
|
|
||||||
|
|
||||||
export const newMocks = () => {
|
|
||||||
return {
|
|
||||||
logger: newLoggerRepositoryMock(),
|
|
||||||
storage: newStorageRepositoryMock(),
|
|
||||||
metricService: newMetricServiceMock(),
|
|
||||||
wideContext: newWideContextRepositoryMock(),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export type Mocks = ReturnType<typeof newMocks>;
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
||||||
import { Readable } from 'node:stream';
|
|
||||||
import { text } from 'node:stream/consumers';
|
|
||||||
import { ReadableStream } from 'node:stream/web';
|
|
||||||
import { StorageRepository } from 'src/repositories/storage.repository';
|
|
||||||
|
|
||||||
describe('StorageRepository (e2e)', () => {
|
|
||||||
let sut: StorageRepository;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
sut = new StorageRepository();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('works', async () => {
|
|
||||||
const Bucket = randomUUID();
|
|
||||||
const Key = randomUUID();
|
|
||||||
|
|
||||||
const contents = 'test object';
|
|
||||||
|
|
||||||
await expect(sut.checkBucket(Bucket)).resolves.toBe(false);
|
|
||||||
await sut.createBucket(Bucket);
|
|
||||||
await expect(sut.checkBucket(Bucket)).resolves.toBe(true);
|
|
||||||
|
|
||||||
await expect(sut.headObject(Bucket, Key)).rejects.toThrow();
|
|
||||||
await sut.putObject(Bucket, Key, Readable.from(contents), false);
|
|
||||||
await sut.headObject(Bucket, Key);
|
|
||||||
|
|
||||||
await expect(sut.listObjects(Bucket, Key)).resolves.toEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
Contents: expect.arrayContaining([expect.objectContaining({ Key, Size: contents.length })]),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
const object = await sut.getObject(Bucket, Key);
|
|
||||||
const stream = Readable.fromWeb(object.Body!.transformToWebStream() as ReadableStream);
|
|
||||||
await expect(text(stream!)).resolves.toBe(contents);
|
|
||||||
|
|
||||||
await sut.deleteObject(Bucket, Key);
|
|
||||||
await expect(sut.headObject(Bucket, Key)).rejects.toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('respects worm', async () => {
|
|
||||||
const Bucket = randomUUID();
|
|
||||||
const Key = randomUUID();
|
|
||||||
|
|
||||||
const contents = 'test object';
|
|
||||||
|
|
||||||
await sut.createBucket(Bucket);
|
|
||||||
await sut.putObject(Bucket, Key, Readable.from(contents), false);
|
|
||||||
await expect(sut.putObject(Bucket, Key, Readable.from(contents), true)).rejects.toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('checks hash', async () => {
|
|
||||||
const Bucket = randomUUID();
|
|
||||||
const Key = randomUUID();
|
|
||||||
|
|
||||||
const contents = randomBytes(5_000_000);
|
|
||||||
|
|
||||||
const hash = createHash('sha256');
|
|
||||||
hash.update(contents);
|
|
||||||
|
|
||||||
await sut.createBucket(Bucket);
|
|
||||||
await sut.putObject(Bucket, Key, Readable.from(contents), false, hash.digest('hex'));
|
|
||||||
await sut.deleteObject(Bucket, Key);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects invalid hash', async () => {
|
|
||||||
const Bucket = randomUUID();
|
|
||||||
const Key = randomUUID();
|
|
||||||
|
|
||||||
const contents = randomBytes(5_000_000);
|
|
||||||
|
|
||||||
const hash = createHash('sha256');
|
|
||||||
hash.update('invalid');
|
|
||||||
|
|
||||||
await sut.createBucket(Bucket);
|
|
||||||
await expect(sut.putObject(Bucket, Key, Readable.from(contents), false, hash.digest('hex'))).rejects.toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('checks hash where files are chunked', async () => {
|
|
||||||
const Bucket = randomUUID();
|
|
||||||
const Key = randomUUID();
|
|
||||||
|
|
||||||
const contents = randomBytes(20_000_000);
|
|
||||||
|
|
||||||
const hash = createHash('sha256');
|
|
||||||
hash.update(contents);
|
|
||||||
|
|
||||||
await sut.createBucket(Bucket);
|
|
||||||
await sut.putObject(Bucket, Key, Readable.from(contents), false, hash.digest('hex'));
|
|
||||||
await sut.deleteObject(Bucket, Key);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"extends": "./tsconfig.json",
|
|
||||||
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"module": "nodenext",
|
|
||||||
"moduleResolution": "nodenext",
|
|
||||||
"resolvePackageJsonExports": true,
|
|
||||||
"esModuleInterop": true,
|
|
||||||
"isolatedModules": true,
|
|
||||||
"declaration": true,
|
|
||||||
"removeComments": true,
|
|
||||||
"emitDecoratorMetadata": true,
|
|
||||||
"experimentalDecorators": true,
|
|
||||||
"allowSyntheticDefaultImports": true,
|
|
||||||
"target": "ES2023",
|
|
||||||
"sourceMap": true,
|
|
||||||
"outDir": "./dist",
|
|
||||||
"incremental": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"strictNullChecks": true,
|
|
||||||
"forceConsistentCasingInFileNames": true,
|
|
||||||
"noImplicitAny": true,
|
|
||||||
"strictBindCallApply": false,
|
|
||||||
"noFallthroughCasesInSwitch": false,
|
|
||||||
"paths": {
|
|
||||||
"src/*": ["./src/*"]
|
|
||||||
},
|
|
||||||
"types": ["jest"]
|
|
||||||
},
|
|
||||||
"include": ["src", "test"]
|
|
||||||
}
|
|
||||||
Generated
-1353
File diff suppressed because it is too large
Load Diff
@@ -11,8 +11,6 @@ onlyBuiltDependencies:
|
|||||||
- unrs-resolver
|
- unrs-resolver
|
||||||
|
|
||||||
catalog:
|
catalog:
|
||||||
'@aws-sdk/client-s3': ^3.965.0
|
|
||||||
'@aws-sdk/lib-storage': ^3.965.0
|
|
||||||
'@better-svelte-email/cli': ^2.1.3
|
'@better-svelte-email/cli': ^2.1.3
|
||||||
'@better-svelte-email/components': ^2.1.3
|
'@better-svelte-email/components': ^2.1.3
|
||||||
'@better-svelte-email/server': ^2.1.3
|
'@better-svelte-email/server': ^2.1.3
|
||||||
|
|||||||
@@ -37,11 +37,6 @@
|
|||||||
"path": "packages/mock-oidc-provider/package.json",
|
"path": "packages/mock-oidc-provider/package.json",
|
||||||
"jsonpath": "$.version"
|
"jsonpath": "$.version"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"type": "json",
|
|
||||||
"path": "packages/restic-api/package.json",
|
|
||||||
"jsonpath": "$.version"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"type": "json",
|
"type": "json",
|
||||||
"path": "packages/web/package.json",
|
"path": "packages/web/package.json",
|
||||||
|
|||||||
Reference in New Issue
Block a user