immich-app/geoshenanigans
1
0
Fork 0
mirror of https://github.com/immich-app/geoshenanigans.git synced 2026-09-30 13:23:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: serve styles from tileserver (#33)

Browse Source
This commit is contained in:
Zack Pollard
2024-09-17 12:00:28 +01:00
committed by GitHub
parent 52b7dc2c70
commit ecce91b478
28 changed files with 7392 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tiles-file.yml
+27 -3
View File
@@ -9,6 +9,7 @@ on:
branches: [ "main" ]
paths:
- ".github/workflows/tiles-file.yml"
- "map-styles/**"
concurrency:
group: ${{ github.workflow }}
@@ -16,14 +17,33 @@ concurrency:
jobs:
deploy_tiles:
name: Deploy Tiles
name: Deploy Tiles & Styles
strategy:
matrix:
bucket_region: [ "wnam", "enam", "weur", "eeur", "apac" ]
defaults:
run:
working-directory: 'map-styles'
runs-on: ubuntu-latest
outputs:
KEY: ${{ steps.latest_build.outputs.KEY }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version-file: 'map-styles/.nvmrc'
- name: Install dependencies
run: npm ci
- name: Generate styles
run: npm run generate
- name: Find latest pmtiles build
id: latest_build
env:
@@ -34,6 +54,7 @@ jobs:
else
build=$FILE_NAME
fi
export build=${build%.pmtiles}
echo "KEY=$build"
echo "KEY=$build" >> $GITHUB_OUTPUT
@@ -53,8 +74,11 @@ jobs:
type = http
url = https://build.protomaps.com/
- name: Copy latest styles to R2
run: rclone --progress copy ./dist/ r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }}/styles/ --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
- name: Copy latest tiles to R2
run: rclone --progress copyto pmtiles:/${{ steps.latest_build.outputs.KEY }} r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }} --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
run: rclone --progress copyto pmtiles:/${{ steps.latest_build.outputs.KEY }}.pmtiles r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }}/tiles.pmtiles --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
update_tile_ref:
name: Update tile server file reference
needs: [deploy_tiles]
@@ -74,7 +98,7 @@ jobs:
- name: Update deployment file name
run: |
sed -i -E 's/"pmtiles_file_name": "(.*)"/"pmtiles_file_name": "${{ needs.deploy_tiles.outputs.KEY }}"/' deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json
sed -i -E 's/"pmtiles_deployment_key": "(.*)"/"pmtiles_deployment_key": "${{ needs.deploy_tiles.outputs.KEY }}"/' deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json
- name: Push updated tiles.tfvars to repo
uses: EndBug/add-and-commit@v9
.github/workflows/tiles-worker.yml
+1 -1
View File
@@ -191,7 +191,7 @@ jobs:
# Figure out how to extract this from terraform at some point or get it into github vars
KV_NAMESPACE_ID: 5a4b82694e8b490db8b8904cdaea4f00
BUCKET_KEY: tiles-weur
FILE_NAME: ${{ fromJson(env.TILES_JSON).pmtiles_file_name }}
DEPLOYMENT_KEY: ${{ fromJson(env.TILES_JSON).pmtiles_deployment_key }}
run: npm run kv:warm
deploy-terragrunt:
.gitmodules
+3
View File
@@ -0,0 +1,3 @@
[submodule "submodules/basemaps"]
path = submodules/basemaps
url = https://github.com/protomaps/basemaps.git
deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
"pmtiles_file_name": "20240812.pmtiles"
"pmtiles_deployment_key": "20240812"
}
deployment/modules/cloudflare/tiles-worker/variables.tf
+1 -1
View File
@@ -1,6 +1,6 @@
variable "cloudflare_account_id" {}
variable "tf_state_postgres_conn_str" {}
variable "pmtiles_file_name" {}
variable "pmtiles_deployment_key" {}
variable "tiles_build_dir" {}
variable "vmetrics_api_token" {}
variable "env" {}
deployment/modules/cloudflare/tiles-worker/workers.tf
+2 -2
View File
@@ -9,8 +9,8 @@ resource "cloudflare_workers_script" "tiles" {
module = true
plain_text_binding {
name = "PMTILES_FILE_NAME"
text = var.pmtiles_file_name
name = "DEPLOYMENT_KEY"
text = var.pmtiles_deployment_key
}
plain_text_binding {
map-styles/.gitignore
+2
View File
@@ -0,0 +1,2 @@
dist
node_modules
map-styles/.nvmrc
+1
View File
@@ -0,0 +1 @@
20.10
map-styles/package-lock.json
Generated
+759
View File
@@ -0,0 +1,759 @@
{
"name": "protomaps-themes-base",
"version": "3.1.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "protomaps-themes-base",
"version": "3.1.0",
"license": "BSD-3-Clause",
"devDependencies": {
"@types/node": "^20.16.5",
"tsx": "^4.7.2"
}
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz",
"integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==",
"cpu": [
"ppc64"
],
"dev": true,
"optional": true,
"os": [
"aix"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz",
"integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz",
"integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz",
"integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz",
"integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz",
"integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz",
"integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz",
"integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz",
"integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz",
"integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ia32": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz",
"integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz",
"integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==",
"cpu": [
"loong64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-mips64el": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz",
"integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==",
"cpu": [
"mips64el"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ppc64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz",
"integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==",
"cpu": [
"ppc64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-riscv64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz",
"integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==",
"cpu": [
"riscv64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-s390x": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz",
"integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==",
"cpu": [
"s390x"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz",
"integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz",
"integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz",
"integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz",
"integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/sunos-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz",
"integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz",
"integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-ia32": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz",
"integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz",
"integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@types/node": {
"version": "20.16.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.5.tgz",
"integrity": "sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==",
"dev": true,
"dependencies": {
"undici-types": "~6.19.2"
}
},
"node_modules/esbuild": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz",
"integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==",
"dev": true,
"hasInstallScript": true,
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"@esbuild/aix-ppc64": "0.23.1",
"@esbuild/android-arm": "0.23.1",
"@esbuild/android-arm64": "0.23.1",
"@esbuild/android-x64": "0.23.1",
"@esbuild/darwin-arm64": "0.23.1",
"@esbuild/darwin-x64": "0.23.1",
"@esbuild/freebsd-arm64": "0.23.1",
"@esbuild/freebsd-x64": "0.23.1",
"@esbuild/linux-arm": "0.23.1",
"@esbuild/linux-arm64": "0.23.1",
"@esbuild/linux-ia32": "0.23.1",
"@esbuild/linux-loong64": "0.23.1",
"@esbuild/linux-mips64el": "0.23.1",
"@esbuild/linux-ppc64": "0.23.1",
"@esbuild/linux-riscv64": "0.23.1",
"@esbuild/linux-s390x": "0.23.1",
"@esbuild/linux-x64": "0.23.1",
"@esbuild/netbsd-x64": "0.23.1",
"@esbuild/openbsd-arm64": "0.23.1",
"@esbuild/openbsd-x64": "0.23.1",
"@esbuild/sunos-x64": "0.23.1",
"@esbuild/win32-arm64": "0.23.1",
"@esbuild/win32-ia32": "0.23.1",
"@esbuild/win32-x64": "0.23.1"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/get-tsconfig": {
"version": "4.8.1",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz",
"integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==",
"dev": true,
"dependencies": {
"resolve-pkg-maps": "^1.0.0"
},
"funding": {
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
}
},
"node_modules/resolve-pkg-maps": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
"dev": true,
"funding": {
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
}
},
"node_modules/tsx": {
"version": "4.19.1",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.1.tgz",
"integrity": "sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==",
"dev": true,
"dependencies": {
"esbuild": "~0.23.0",
"get-tsconfig": "^4.7.5"
},
"bin": {
"tsx": "dist/cli.mjs"
},
"engines": {
"node": ">=18.0.0"
},
"optionalDependencies": {
"fsevents": "~2.3.3"
}
},
"node_modules/undici-types": {
"version": "6.19.8",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
"dev": true
}
},
"dependencies": {
"@esbuild/aix-ppc64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz",
"integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==",
"dev": true,
"optional": true
},
"@esbuild/android-arm": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz",
"integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==",
"dev": true,
"optional": true
},
"@esbuild/android-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz",
"integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==",
"dev": true,
"optional": true
},
"@esbuild/android-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz",
"integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==",
"dev": true,
"optional": true
},
"@esbuild/darwin-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz",
"integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==",
"dev": true,
"optional": true
},
"@esbuild/darwin-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz",
"integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==",
"dev": true,
"optional": true
},
"@esbuild/freebsd-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz",
"integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==",
"dev": true,
"optional": true
},
"@esbuild/freebsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz",
"integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==",
"dev": true,
"optional": true
},
"@esbuild/linux-arm": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz",
"integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==",
"dev": true,
"optional": true
},
"@esbuild/linux-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz",
"integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==",
"dev": true,
"optional": true
},
"@esbuild/linux-ia32": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz",
"integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==",
"dev": true,
"optional": true
},
"@esbuild/linux-loong64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz",
"integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==",
"dev": true,
"optional": true
},
"@esbuild/linux-mips64el": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz",
"integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==",
"dev": true,
"optional": true
},
"@esbuild/linux-ppc64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz",
"integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==",
"dev": true,
"optional": true
},
"@esbuild/linux-riscv64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz",
"integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==",
"dev": true,
"optional": true
},
"@esbuild/linux-s390x": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz",
"integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==",
"dev": true,
"optional": true
},
"@esbuild/linux-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz",
"integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==",
"dev": true,
"optional": true
},
"@esbuild/netbsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz",
"integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==",
"dev": true,
"optional": true
},
"@esbuild/openbsd-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz",
"integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==",
"dev": true,
"optional": true
},
"@esbuild/openbsd-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz",
"integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==",
"dev": true,
"optional": true
},
"@esbuild/sunos-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz",
"integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==",
"dev": true,
"optional": true
},
"@esbuild/win32-arm64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz",
"integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==",
"dev": true,
"optional": true
},
"@esbuild/win32-ia32": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz",
"integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==",
"dev": true,
"optional": true
},
"@esbuild/win32-x64": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz",
"integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==",
"dev": true,
"optional": true
},
"@types/node": {
"version": "20.16.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.5.tgz",
"integrity": "sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==",
"dev": true,
"requires": {
"undici-types": "~6.19.2"
}
},
"esbuild": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz",
"integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==",
"dev": true,
"requires": {
"@esbuild/aix-ppc64": "0.23.1",
"@esbuild/android-arm": "0.23.1",
"@esbuild/android-arm64": "0.23.1",
"@esbuild/android-x64": "0.23.1",
"@esbuild/darwin-arm64": "0.23.1",
"@esbuild/darwin-x64": "0.23.1",
"@esbuild/freebsd-arm64": "0.23.1",
"@esbuild/freebsd-x64": "0.23.1",
"@esbuild/linux-arm": "0.23.1",
"@esbuild/linux-arm64": "0.23.1",
"@esbuild/linux-ia32": "0.23.1",
"@esbuild/linux-loong64": "0.23.1",
"@esbuild/linux-mips64el": "0.23.1",
"@esbuild/linux-ppc64": "0.23.1",
"@esbuild/linux-riscv64": "0.23.1",
"@esbuild/linux-s390x": "0.23.1",
"@esbuild/linux-x64": "0.23.1",
"@esbuild/netbsd-x64": "0.23.1",
"@esbuild/openbsd-arm64": "0.23.1",
"@esbuild/openbsd-x64": "0.23.1",
"@esbuild/sunos-x64": "0.23.1",
"@esbuild/win32-arm64": "0.23.1",
"@esbuild/win32-ia32": "0.23.1",
"@esbuild/win32-x64": "0.23.1"
}
},
"fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"optional": true
},
"get-tsconfig": {
"version": "4.8.1",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz",
"integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==",
"dev": true,
"requires": {
"resolve-pkg-maps": "^1.0.0"
}
},
"resolve-pkg-maps": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
"dev": true
},
"tsx": {
"version": "4.19.1",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.1.tgz",
"integrity": "sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==",
"dev": true,
"requires": {
"esbuild": "~0.23.0",
"fsevents": "~2.3.3",
"get-tsconfig": "^4.7.5"
}
},
"undici-types": {
"version": "6.19.8",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
"dev": true
}
}
}
map-styles/package.json
+15
View File
@@ -0,0 +1,15 @@
{
"name": "immich-map-styles",
"description": "Generate immich style variations from the protomaps styles",
"main": "index.ts",
"scripts": {
"generate": "tsx src/index.ts"
},
"author": "The Protomaps Authors",
"license": "BSD-3-Clause",
"homepage": "https://github.com/protomaps/basemaps",
"devDependencies": {
"tsx": "^4.7.2",
"@types/node": "^20.16.5"
}
}
map-styles/src/base.json
+13
View File
@@ -0,0 +1,13 @@
{
"version": 9,
"name": "Immich Map",
"id": "immich-map",
"sources": {
"tiles": {
"type": "vector",
"url": "https://tiles.immich.cloud/v1.json"
}
},
"sprite": "https://static.immich.cloud/tiles/sprites/v1/light",
"glyphs": "https://static.immich.cloud/tiles/fonts/{fontstack}/{range}.pbf"
}
map-styles/src/index.ts
+35
View File
@@ -0,0 +1,35 @@
import themes, { ImmichTheme } from './themes';
import fs from 'fs';
import path from 'path';
import { layersWithCustomTheme } from '../../submodules/basemaps/styles/src';
const generateLayers = (themeName: string) => {
const sourceName = 'protomaps'
const theme = themes[themeName];
const layers = layersWithCustomTheme(sourceName, theme);
const style = {
version: 8,
"name": "Immich Map",
"id": `immich-map-${themeName}`,
sources: {
[sourceName]: {
type: "vector",
url: 'https://tiles.immich.cloud/v1.json',
},
},
layers: layers,
sprite: `https://static.immich.cloud/tiles/sprites/v1/${theme.iconStyle}`,
glyphs: "https://static.immich.cloud/tiles/fonts/{fontstack}/{range}.pbf",
};
return style;
}
const dist = path.resolve(__dirname, '../dist');
if (!fs.existsSync(dist)) {
fs.mkdirSync(dist);
}
for (const theme of Object.keys(themes)) {
fs.writeFileSync(path.resolve(dist, `${theme}.json`), JSON.stringify(generateLayers(theme), null, 2));
}
map-styles/src/themes.ts
+23
View File
@@ -0,0 +1,23 @@
import { BLACK, LIGHT, Theme } from '../../submodules/basemaps/styles/src/themes';
export interface ImmichTheme extends Theme {
iconStyle: 'dark' | 'light';
}
export const IMMICH_LIGHT: ImmichTheme = {
iconStyle: 'light',
...LIGHT,
water: 'rgba(148, 209, 236, 0.66)'
};
export const IMMICH_DARK: ImmichTheme = {
iconStyle: 'dark',
...BLACK,
};
const themes: Record<string, ImmichTheme> = {
light: IMMICH_LIGHT,
dark: IMMICH_DARK,
};
export default themes;
map-styles/tsconfig.json
+13
View File
@@ -0,0 +1,13 @@
{
"compilerOptions": {
"target": "es6",
"lib": ["es2020", "dom"],
"strict": true,
"moduleResolution": "node",
"paths": {
},
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
},
"include": ["src/*.ts"]
}
submodules/basemaps
Submodule
+1
Submodule submodules/basemaps added at 217103cf64
tiles/src/env.d.ts
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
interface WorkerEnv extends Omit<Env, 'ENVIRONMENT' | 'PMTILES_FILE_NAME'> {
interface WorkerEnv extends Omit<Env, 'ENVIRONMENT' | 'DEPLOYMENT_KEY'> {
ENVIRONMENT: string;
PMTILES_FILE_NAME: string;
DEPLOYMENT_KEY: string;
VMETRICS_API_TOKEN: string;
}
tiles/src/index.ts
+35 -6
View File
@@ -17,13 +17,20 @@ declare global {
}
/* eslint-enable no-var */
const URL_MATCHER = /^\/v(?<VERSION>[0-9]+)((?=)|(?<JSON>\.json)|\/(?<Z>\d+)\/(?<X>\d+)\/(?<Y>\d+).mvt)$/;
const URL_MATCHER =
/^\/v(?<VERSION>[0-9]+)((?=)|\/style\/(?<STYLE>\w+)(?:\.json)?|(?<JSON>\.json)|\/(?<Z>\d+)\/(?<X>\d+)\/(?<Y>\d+).mvt)$/;
type PMTilesParams = {
requestType: 'tile' | 'json' | undefined;
requestType: 'tile' | 'json' | 'style' | undefined;
url: URL;
};
type PMTilesStyleParams = PMTilesParams & {
requestType: 'style';
version: string;
style: string;
};
type PMTilesTileParams = PMTilesParams & {
requestType: 'tile';
version: string;
@@ -38,7 +45,7 @@ type PMTilesJsonParams = PMTilesParams & {
};
enum Header {
PMTILES_FILE_IDENTIFIER = 'PMTiles-File-Identifier',
PMTILES_DEPLOYMENT_KEY = 'PMTiles-Deployment-Key',
CACHE_CONTROL = 'Cache-Control',
ACCESS_CONTROL_ALLOW_ORIGIN = 'Access-Control-Allow-Origin',
VARY = 'Vary',
@@ -51,11 +58,14 @@ export function parseUrl(request: Request): PMTilesParams {
const url = new URL(request.url);
const matches = URL_MATCHER.exec(url.pathname);
const version = matches?.groups?.VERSION;
const style = matches?.groups?.STYLE;
const z = matches?.groups?.Z;
const x = matches?.groups?.X;
const y = matches?.groups?.Y;
if (version && z && x && y) {
return { requestType: 'tile', version, z, x, y, url } as PMTilesTileParams;
} else if (style) {
return { requestType: 'style', style, url } as PMTilesStyleParams;
} else if (version) {
return { requestType: 'json', version, url } as PMTilesJsonParams;
}
@@ -94,6 +104,16 @@ async function handleRequest(
return cacheResponse(new Response(JSON.stringify(tileJson), { headers: respHeaders, status: 200 }));
}
async function handleStyleRequest(respHeaders: Headers) {
const { style } = pmTilesParams as PMTilesStyleParams;
const styleJson = await storageRepository.getAsStream('styles/' + style + '.json');
if (!styleJson) {
return cacheResponse(new Response('Style not found', { status: 404 }));
}
respHeaders.set(Header.CONTENT_TYPE, 'application/json');
return cacheResponse(new Response(styleJson, { headers: respHeaders, status: 200 }));
}
if (request.method.toUpperCase() !== 'GET') {
return new Response(undefined, { status: 405 });
}
@@ -102,7 +122,7 @@ async function handleRequest(
const cached = await metrics.monitorAsyncFunction({ name: 'match_request_from_cdn' }, (url) => cache.match(url))(
request.url,
);
if (cached && cached.headers.get(Header.PMTILES_FILE_IDENTIFIER) === env.PMTILES_FILE_NAME) {
if (cached && cached.headers.get(Header.PMTILES_DEPLOYMENT_KEY) === env.DEPLOYMENT_KEY) {
const cacheHeaders = new Headers(cached.headers);
const encodeBody = cacheHeaders.has('content-encoding') ? 'manual' : 'automatic';
return new Response(cached.body, {
@@ -132,7 +152,7 @@ async function handleRequest(
Object.entries(bucketMap).filter(([key]) => buckets.includes(key as R2BucketRegion)),
);
const storageRepository = new R2StorageRepository(filteredBucketMap, env.PMTILES_FILE_NAME, metrics);
const storageRepository = new R2StorageRepository(filteredBucketMap, env.DEPLOYMENT_KEY, metrics);
const pmTilesService = await metrics.monitorAsyncFunction({ name: 'pmtiles_init' }, PMTilesService.init)(
storageRepository,
memCacheRepository,
@@ -144,7 +164,7 @@ async function handleRequest(
respHeaders.set(Header.CACHE_CONTROL, `public, max-age=${60 * 60 * 24 * 31}`);
respHeaders.set(Header.ACCESS_CONTROL_ALLOW_ORIGIN, '*');
respHeaders.set(Header.VARY, 'Origin');
respHeaders.set(Header.PMTILES_FILE_IDENTIFIER, env.PMTILES_FILE_NAME);
respHeaders.set(Header.PMTILES_DEPLOYMENT_KEY, env.DEPLOYMENT_KEY);
const pmTilesParams = parseUrl(request);
@@ -164,6 +184,15 @@ async function handleRequest(
handleJsonRequest,
)(respHeaders);
}
if (pmTilesParams.requestType === 'style') {
const { style } = pmTilesParams as PMTilesStyleParams;
console.log('style');
return await metrics.monitorAsyncFunction(
{ name: 'style_request', tags: { style } },
handleStyleRequest,
)(respHeaders);
}
} catch (e) {
console.error(e);
return new Response('Internal Server Error', { status: 500 });
tiles/src/interface.ts
+5 -3
View File
@@ -11,9 +11,11 @@ export interface IMemCacheRepository {
}
export interface IStorageRepository {
get(range: { length: number; offset: number }): Promise<ArrayBuffer>;
getAsStream(range: { length: number; offset: number }): Promise<ReadableStream>;
getFileName(): string;
getRange(range: { length: number; offset: number }): Promise<ArrayBuffer>;
getRangeAsStream(range: { length: number; offset: number }): Promise<ReadableStream>;
get(key: string): Promise<ArrayBuffer>;
getAsStream(key: string): Promise<ReadableStream>;
getDeploymentKey(): string;
}
export interface IDeferredRepository {
tiles/src/kv-warmer.ts
+10 -7
View File
@@ -43,7 +43,7 @@ export function decompress(buf: ArrayBuffer, compression: Compression): ArrayBuf
}
const getDirectory = async (length: number, offset: number, source: IStorageRepository, header: Header) => {
const resp = await source.get({ offset, length });
const resp = await source.getRange({ offset, length });
const data = decompress(resp, header.internalCompression);
const entries = deserializeIndex(await new Response(data).arrayBuffer());
if (entries.length === 0) {
@@ -62,7 +62,7 @@ const handler = async () => {
CLOUDFLARE_ACCOUNT_ID,
KV_NAMESPACE_ID,
BUCKET_KEY,
FILE_NAME,
DEPLOYMENT_KEY,
} = process.env;
if (
!S3_ACCESS_KEY ||
@@ -72,7 +72,7 @@ const handler = async () => {
!CLOUDFLARE_ACCOUNT_ID ||
!KV_NAMESPACE_ID ||
!BUCKET_KEY ||
!FILE_NAME
!DEPLOYMENT_KEY
) {
throw new Error('Missing environment variables');
}
@@ -87,7 +87,7 @@ const handler = async () => {
},
});
const storageRepository = new S3StorageRepository(client, BUCKET_KEY, FILE_NAME);
const storageRepository = new S3StorageRepository(client, BUCKET_KEY, DEPLOYMENT_KEY);
const memCacheRepository = new MemCacheRepository(new Map());
const kvRepository = new FakeKVRepository();
const metricsRepository = new FakeMetricsRepository();
@@ -99,7 +99,7 @@ const handler = async () => {
);
console.log('Checking if already warmed');
const kvCheckKey = encodeURIComponent(`${FILE_NAME}|kv-warmed`);
const kvCheckKey = encodeURIComponent(`${DEPLOYMENT_KEY}|kv-warmed`);
console.log(kvCheckKey);
console.log(
'url',
@@ -189,7 +189,7 @@ const handler = async () => {
header,
);
const stream = DirectoryStream.fromDirectory(directory);
const cacheKey = getDirectoryCacheKey(FILE_NAME, {
const cacheKey = getDirectoryCacheKey(DEPLOYMENT_KEY, {
offset: entry.offset + header.leafDirectoryOffset,
length: entry.length,
});
@@ -237,4 +237,7 @@ process.on('uncaughtException', (e) => {
handler()
.then(() => console.log('Done'))
.catch((e) => console.error(e));
.catch((e) => {
console.error('Error', e);
throw e;
});
tiles/src/pmtiles/pmtiles.service.ts
+8 -8
View File
@@ -122,14 +122,14 @@ export class PMTilesService {
metrics: IMetricsRepository,
): Promise<PMTilesService> {
const p = new PMTilesService(source, memCache, kvCache, metrics);
const headerCacheKey = getHeaderCacheKey(source.getFileName());
const headerCacheKey = getHeaderCacheKey(source.getDeploymentKey());
if (memCache.get(headerCacheKey)) {
return p;
}
const [header, root] = await p.getHeaderAndRootFromSource();
memCache.set(headerCacheKey, header);
memCache.set(
getDirectoryCacheKey(source.getFileName(), {
getDirectoryCacheKey(source.getDeploymentKey(), {
offset: header.rootDirectoryOffset,
length: header.rootDirectoryLength,
}),
@@ -139,7 +139,7 @@ export class PMTilesService {
}
private getHeader(): Header {
const key = getHeaderCacheKey(this.source.getFileName());
const key = getHeaderCacheKey(this.source.getDeploymentKey());
const memCached = this.memCache.get<Header>(key);
if (!memCached) {
throw new Error('Header not found in cache');
@@ -148,7 +148,7 @@ export class PMTilesService {
}
private getRootDirectory(header: Header): Directory {
const key = getDirectoryCacheKey(this.source.getFileName(), {
const key = getDirectoryCacheKey(this.source.getDeploymentKey(), {
offset: header.rootDirectoryOffset,
length: header.rootDirectoryLength,
});
@@ -160,7 +160,7 @@ export class PMTilesService {
}
async getHeaderAndRootFromSource(): Promise<[Header, Directory]> {
const resp = await this.source.get({ offset: 0, length: 16384 });
const resp = await this.source.getRange({ offset: 0, length: 16384 });
const v = new DataView(resp);
if (v.getUint16(0, true) !== 0x4d50) {
throw new Error('Wrong magic number for PMTiles archive');
@@ -181,7 +181,7 @@ export class PMTilesService {
}
private async getDirectory(offset: number, length: number): Promise<DirectoryStream> {
const cacheKey = getDirectoryCacheKey(this.source.getFileName(), { offset, length });
const cacheKey = getDirectoryCacheKey(this.source.getDeploymentKey(), { offset, length });
console.log(cacheKey);
const kvValueStream = await this.kvCache.getAsStream(cacheKey);
if (kvValueStream) {
@@ -227,7 +227,7 @@ export class PMTilesService {
)(tileId);
}
const tile = await this.metrics.monitorAsyncFunction({ name: 'get_tile' }, (offset, length) =>
this.source.getAsStream({ offset, length }),
this.source.getRangeAsStream({ offset, length }),
)(header.tileDataOffset + offset, length);
return tile;
}
@@ -235,7 +235,7 @@ export class PMTilesService {
private async getMetadata(): Promise<Metadata> {
const header = this.getHeader();
const resp = await this.source.get({
const resp = await this.source.getRange({
offset: header.jsonMetadataOffset,
length: header.jsonMetadataLength,
});
tiles/src/pmtiles/utils.ts
+1 -1
View File
@@ -269,5 +269,5 @@ export function getHeaderCacheKey(archiveName: string): string {
}
export function getDirectoryCacheKey(fileName: string, range: { offset: number; length: number }): string {
return `${fileName}|prodv1|${range.offset}|${range.length}`;
return `${fileName}|${range.offset}|${range.length}`;
}
tiles/src/repository.ts
+44 -22
View File
@@ -42,42 +42,53 @@ export class MemCacheRepository implements IMemCacheRepository {
export class R2StorageRepository implements IStorageRepository {
constructor(
private buckets: { [key: string]: R2Bucket },
private fileName: string,
private deploymentKey: string,
private metrics: IMetricsRepository,
) {}
getFileName(): string {
return this.fileName;
getDeploymentKey(): string {
return this.deploymentKey;
}
private async getR2Object(range: { offset: number; length: number }) {
private async getR2Object(args: { key?: string; range?: { offset: number; length: number } }) {
const { key, range } = args;
const fileName = key ?? 'tiles.pmtiles';
const filePath = `${this.deploymentKey}/${fileName}`;
const metric = Metric.create('r2_storage_get');
const { offset, length } = range;
const { key: bucketKey, resp } = await Promise.race(
Object.entries(this.buckets).map(async ([key, bucket]) => {
const resp = await bucket.get(this.fileName, {
range: { offset, length },
const resp = await bucket.get(filePath, {
range,
});
return { key, resp };
}),
);
metric.durationField(`${bucketKey}_duration`);
metric.addTag('bucket_key', bucketKey);
metric.addTag('file_name', fileName);
metric.intField('count', 1);
this.metrics.push(metric);
if (!resp) {
throw new Error('Archive not found');
throw new Error(`Key not found ${filePath}`);
}
return resp;
}
async get(range: { offset: number; length: number }): Promise<ArrayBuffer> {
return (await this.getR2Object(range)).arrayBuffer();
async getRange(range: { offset: number; length: number }): Promise<ArrayBuffer> {
return (await this.getR2Object({ range })).arrayBuffer();
}
async getAsStream(range: { offset: number; length: number }): Promise<ReadableStream> {
return (await this.getR2Object(range)).body;
async getRangeAsStream(range: { offset: number; length: number }): Promise<ReadableStream> {
return (await this.getR2Object({ range })).body;
}
async get(key: string): Promise<ArrayBuffer> {
return (await this.getR2Object({ key })).arrayBuffer();
}
async getAsStream(key: string): Promise<ReadableStream> {
return (await this.getR2Object({ key })).body;
}
}
@@ -85,14 +96,15 @@ export class S3StorageRepository implements IStorageRepository {
constructor(
private client: S3Client,
private bucketKey: string,
private fileName: string,
private deploymentKey: string,
) {}
private async getS3Object(range: { offset: number; length: number }) {
private async getS3Object(args: { key?: string; range?: { offset: number; length: number } }) {
const { range } = args;
const command = new GetObjectCommand({
Bucket: this.bucketKey,
Key: this.fileName,
Range: `bytes=${range.offset}-${range.offset + range.length - 1}`,
Key: this.deploymentKey + '/' + (args.key ?? 'tiles.pmtiles'),
Range: range ? `bytes=${range.offset}-${range.offset + range.length - 1}` : undefined,
});
const response = await this.client.send(command);
const data = response.Body;
@@ -102,18 +114,28 @@ export class S3StorageRepository implements IStorageRepository {
return data;
}
async get(range: { length: number; offset: number }): Promise<ArrayBuffer> {
const data = await this.getS3Object(range);
async getRange(range: { length: number; offset: number }): Promise<ArrayBuffer> {
const data = await this.getS3Object({ range });
return (await data.transformToByteArray()).buffer;
}
async getAsStream(range: { length: number; offset: number }): Promise<ReadableStream> {
const data = await this.getS3Object(range);
async getRangeAsStream(range: { length: number; offset: number }): Promise<ReadableStream> {
const data = await this.getS3Object({ range });
return data.transformToWebStream();
}
getFileName(): string {
return this.fileName;
async get(key: string): Promise<ArrayBuffer> {
const data = await this.getS3Object({ key });
return (await data.transformToByteArray()).buffer;
}
async getAsStream(key: string): Promise<ReadableStream> {
const data = await this.getS3Object({ key });
return data.transformToWebStream();
}
getDeploymentKey(): string {
return this.deploymentKey;
}
}
tiles/test/assets/styles/dark.json
+3180
View File
File diff suppressed because it is too large Load Diff
tiles/test/assets/styles/light.json
+3180
View File
File diff suppressed because it is too large Load Diff
tiles/test/assets/zoom1.pmtiles → tiles/test/assets/tiles.pmtiles
View File
tiles/test/index.spec.ts
+28 -6
View File
@@ -10,20 +10,27 @@ describe('integration tests', () => {
describe('success - zoom1', () => {
beforeAll(async () => {
const buckets = [env.BUCKET_WEUR, env.BUCKET_WNAM, env.BUCKET_EEUR, env.BUCKET_APAC, env.BUCKET_ENAM];
const files = ['tiles.pmtiles', 'styles/dark.json', 'styles/light.json'];
for (const bucket of buckets) {
const file = await fetch(`http://localhost:${inject('port')}/zoom1.pmtiles`);
if (!file.body) {
throw new Error('File body is undefined');
for (const fileName of files) {
const file = await fetch(`http://localhost:${inject('port')}/${fileName}`);
if (!file.body) {
throw new Error('File body is undefined');
}
const body = file.body as ReadableStream;
await bucket.put(`${env.DEPLOYMENT_KEY}/${fileName}`, body);
console.log('File uploaded');
}
const body = file.body as ReadableStream;
await bucket.put(env.PMTILES_FILE_NAME, body);
console.log('File uploaded');
}
}, 30000);
it('responds with correct json file', async () => {
const response = await SELF.fetch('https://example.com/v1');
await expect(JSON.stringify(await response.json(), null, 2)).toMatchFileSnapshot('./__snapshots__/v1.json');
});
it('response with correct style', async () => {
const response = await SELF.fetch('https://example.com/v1/style/dark');
await expect(JSON.stringify(await response.json(), null, 2)).toMatchFileSnapshot('./assets/styles/dark.json');
});
it('responds with correct tile', async () => {
const response = await SELF.fetch('https://example.com/v1/0/0/0.mvt');
expect(response.status).toBe(200);
@@ -73,6 +80,21 @@ describe('parseUrl', () => {
});
});
it.each`
url | style | version
${'http://example.com/v1/style/dark'} | ${'dark'} | ${'1'}
${'http://example.com/v2/style/light'} | ${'light'} | ${'2'}
${'http://example.com/v2/style/dark.json'} | ${'dark'} | ${'2'}
${'http://example.com/v1/style/light.json'} | ${'light'} | ${'1'}
`('style endpoint $url returns style $style', ({ url, style }) => {
const result = parseUrl(new Request(url));
expect(result).toStrictEqual({
requestType: 'style',
style,
url: new URL(url),
});
});
it.each`
url | version | z | x | y
${'http://example.com/v1/0/0/0.mvt'} | ${'1'} | ${'0'} | ${'0'} | ${'0'}
tiles/worker-configuration.d.ts
Vendored
+1 -1
View File
@@ -3,7 +3,7 @@
interface Env {
KV: KVNamespace;
PMTILES_FILE_NAME: "v1.pmtiles";
DEPLOYMENT_KEY: "20240812";
ENVIRONMENT: "prod";
BUCKET_WNAM: R2Bucket;
BUCKET_ENAM: R2Bucket;
tiles/wrangler.toml
+1 -1
View File
@@ -40,5 +40,5 @@ preview_bucket_name = "tiles-apac"
# Define environment variables for your Workers script
[vars]
PMTILES_FILE_NAME = "v1.pmtiles"
DEPLOYMENT_KEY = "20240812"
ENVIRONMENT = "production"