From bd206d378ac811e586978d3d4c39537092776d13 Mon Sep 17 00:00:00 2001 From: bo0tzz Date: Tue, 8 Jul 2025 14:24:30 +0200 Subject: [PATCH] chore: adapt volsync template to use new r2 bucket module secret (#818) --- kubernetes/apps/tools/outline/ks.yaml | 4 +- templates/kubernetes/volsync/README.md | 9 ++--- .../volsync/replication/externalsecret.yaml | 39 +++++++++++++++++-- .../replication/replicationdestination.yaml | 2 +- .../replication/replicationsource.yaml | 2 +- .../shared/1password/account/secrets.tf | 3 +- .../shared/cloudflare/account/1password.tf | 39 ------------------- .../modules/shared/cloudflare/account/r2.tf | 21 ++++++++-- .../shared/cloudflare/account/terragrunt.hcl | 2 +- .../cloudflare-pages-project/config.tf | 2 +- .../cloudflare-pages-project/locals.tf | 4 +- tf/shared/modules/cloudflare-pages/config.tf | 2 +- tf/shared/modules/cloudflare-pages/domain.tf | 4 +- tf/shared/modules/cloudflare-pages/locals.tf | 6 +-- tf/shared/modules/domain/outputs.tf | 8 ++-- 15 files changed, 76 insertions(+), 71 deletions(-) diff --git a/kubernetes/apps/tools/outline/ks.yaml b/kubernetes/apps/tools/outline/ks.yaml index 7b36dcce..cd692677 100644 --- a/kubernetes/apps/tools/outline/ks.yaml +++ b/kubernetes/apps/tools/outline/ks.yaml @@ -49,4 +49,6 @@ spec: APP: *app VOLSYNC_CAPACITY: 20Gi VOLSYNC_SCHEDULE: "0 17 * * *" - VOLSYNC_REPO_SECRET: mich-cloudflare-r2-outline-volsync-backup + VOLSYNC_SECRET_STORE: 1p-tf + VOLSYNC_RESTIC_PASSWORD_SECRET: OUTLINE_VOLSYNC_BACKUPS_RESTIC_SECRET + VOLSYNC_BUCKET_SECRET: OUTLINE_VOLSYNC_BACKUPS_BUCKET diff --git a/templates/kubernetes/volsync/README.md b/templates/kubernetes/volsync/README.md index 6225a481..e87305bb 100644 --- a/templates/kubernetes/volsync/README.md +++ b/templates/kubernetes/volsync/README.md @@ -43,9 +43,6 @@ resources: - `APP`: The application name - `VOLSYNC_CAPACITY`: The PVC size -- `VOLSYNC_REPO_SECRET` - The name of the 1password entry holding the appropriate secret values: - - `RESTIC_REPOSITORY` - - `RESTIC_PASSWORD` - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` +- `VOLSYNC_SECRET_STORE`: The name of the ClusterSecretStore to use +- `VOLSYNC_RESTIC_PASSWORD_SECRET`: The name of the 1password entry holding the restic password +- `VOLSYNC_BUCKET_SECRET`: The name of the 1password entry holding the bucket credentials diff --git a/templates/kubernetes/volsync/replication/externalsecret.yaml b/templates/kubernetes/volsync/replication/externalsecret.yaml index 63bd1345..078483e8 100644 --- a/templates/kubernetes/volsync/replication/externalsecret.yaml +++ b/templates/kubernetes/volsync/replication/externalsecret.yaml @@ -1,6 +1,37 @@ -apiVersion: onepassword.com/v1 -kind: OnePasswordItem +apiVersion: external-secrets.io/v1 +kind: ExternalSecret metadata: - name: "${VOLSYNC_REPO_SECRET}" + name: ${APP}-volsync-repo spec: - itemPath: "vaults/Kubernetes/items/${VOLSYNC_REPO_SECRET}" + secretStoreRef: + kind: ClusterSecretStore + name: ${VOLSYNC_SECRET_STORE} + refreshInterval: "20s" + target: + template: + engineVersion: v2 + data: + RESTIC_PASSWORD: "{{ .restic_password }}" + RESTIC_REPOSITORY: "s3:{{ .endpoint }}/{{ .bucket_name }}" + AWS_ACCESS_KEY_ID: "{{ .access_key_id }}" + AWS_SECRET_ACCESS_KEY: "{{ .secret_access_key }}" + data: + - secretKey: restic_password + remoteRef: + key: ${VOLSYNC_RESTIC_PASSWORD_SECRET} + - secretKey: access_key_id + remoteRef: + key: ${VOLSYNC_BUCKET_SECRET} + property: access_key_id + - secretKey: secret_access_key + remoteRef: + key: ${VOLSYNC_BUCKET_SECRET} + property: secret_access_key + - secretKey: bucket_name + remoteRef: + key: ${VOLSYNC_BUCKET_SECRET} + property: bucket_name + - secretKey: endpoint + remoteRef: + key: ${VOLSYNC_BUCKET_SECRET} + property: endpoint diff --git a/templates/kubernetes/volsync/replication/replicationdestination.yaml b/templates/kubernetes/volsync/replication/replicationdestination.yaml index b0e89593..8bc522b8 100644 --- a/templates/kubernetes/volsync/replication/replicationdestination.yaml +++ b/templates/kubernetes/volsync/replication/replicationdestination.yaml @@ -9,7 +9,7 @@ spec: manual: restore-once restic: copyMethod: Snapshot - repository: "${VOLSYNC_REPO_SECRET}" + repository: "${APP}-volsync-repo" cacheStorageClassName: "zfs" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}" storageClassName: "zfs" diff --git a/templates/kubernetes/volsync/replication/replicationsource.yaml b/templates/kubernetes/volsync/replication/replicationsource.yaml index dc63bb81..ad51fac9 100644 --- a/templates/kubernetes/volsync/replication/replicationsource.yaml +++ b/templates/kubernetes/volsync/replication/replicationsource.yaml @@ -10,7 +10,7 @@ spec: schedule: "${VOLSYNC_SCHEDULE:-0 4 * * *}" restic: copyMethod: Clone - repository: "${VOLSYNC_REPO_SECRET}" + repository: "${APP}-volsync-repo" cacheStorageClassName: "zfs" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-1Gi}" storageClassName: "zfs" diff --git a/tf/deployment/modules/shared/1password/account/secrets.tf b/tf/deployment/modules/shared/1password/account/secrets.tf index 4423adc5..32f63002 100644 --- a/tf/deployment/modules/shared/1password/account/secrets.tf +++ b/tf/deployment/modules/shared/1password/account/secrets.tf @@ -46,7 +46,8 @@ module "generated-secrets" { { name = "PREVIEWS_GITHUB_WEBHOOK_SECRET" }, { name = "AUTH_ZITADEL_MASTER_KEY", length = 32 }, { name = "OUTLINE_SECRET_KEY", length = 64, type = "numeric" }, - { name = "OUTLINE_UTILS_SECRET" } + { name = "OUTLINE_UTILS_SECRET" }, + { name = "OUTLINE_VOLSYNC_BACKUPS_RESTIC_SECRET" } ] } } diff --git a/tf/deployment/modules/shared/cloudflare/account/1password.tf b/tf/deployment/modules/shared/cloudflare/account/1password.tf index 2a66e3a0..4d05b9f6 100644 --- a/tf/deployment/modules/shared/cloudflare/account/1password.tf +++ b/tf/deployment/modules/shared/cloudflare/account/1password.tf @@ -95,45 +95,6 @@ resource "onepassword_item" "mich_cloudflare_r2_data_pipeline_vmetrics_backups_b } } -resource "random_password" "outline_backups_restic_secret" { - length = 40 - special = true - override_special = "!@#$%^&*()_+" -} - -resource "onepassword_item" "mich_cloudflare_r2_outline_volsync_backup" { - vault = data.onepassword_vault.kubernetes.uuid - title = "mich-cloudflare-r2-outline-volsync-backup" - category = "secure_note" - section { - label = "Cloudflare R2 Bucket" - - field { - label = "RESTIC_REPOSITORY" - type = "STRING" - value = "s3:https://${cloudflare_r2_bucket.outline_volsync_backups.account_id}.r2.cloudflarestorage.com/${cloudflare_r2_bucket.outline_volsync_backups.name}" - } - - field { - label = "RESTIC_PASSWORD" - type = "CONCEALED" - value = random_password.outline_backups_restic_secret.result - } - - field { - label = "AWS_ACCESS_KEY_ID" - type = "CONCEALED" - value = data.terraform_remote_state.api_keys_state.outputs.mich_cloudflare_r2_token_id - } - - field { - label = "AWS_SECRET_ACCESS_KEY" - type = "CONCEALED" - value = sha256(data.terraform_remote_state.api_keys_state.outputs.mich_cloudflare_r2_token_value) - } - } -} - resource "onepassword_item" "mich_cloudflare_r2_outline_database_backups_bucket" { vault = data.onepassword_vault.kubernetes.uuid title = "mich-cloudflare-r2-outline-database-backup-bucket" diff --git a/tf/deployment/modules/shared/cloudflare/account/r2.tf b/tf/deployment/modules/shared/cloudflare/account/r2.tf index 6e2ea86c..8214549f 100644 --- a/tf/deployment/modules/shared/cloudflare/account/r2.tf +++ b/tf/deployment/modules/shared/cloudflare/account/r2.tf @@ -1,3 +1,7 @@ +data "onepassword_vault" "tf" { + name = "tf" +} + resource "cloudflare_r2_bucket" "tf_state_database_backups" { account_id = var.cloudflare_account_id name = "tf-state-database-backups" @@ -22,10 +26,19 @@ resource "cloudflare_r2_bucket" "outline_database_backups" { location = "WEUR" } -resource "cloudflare_r2_bucket" "outline_volsync_backups" { - account_id = var.cloudflare_account_id - name = "outline-volsync-backups" - location = "WEUR" +moved { + from = cloudflare_r2_bucket.outline_volsync_backups + to = module.outline_volsync_backups.cloudflare_r2_bucket.bucket +} + +module "outline_volsync_backups" { + source = "./shared/modules/cloudflare-r2-bucket" + + bucket_name = "outline-volsync-backups" + cloudflare_account_id = var.cloudflare_account_id + onepassword_vault_id = data.onepassword_vault.tf.uuid + item_name = "OUTLINE_VOLSYNC_BACKUPS_BUCKET" + allowed_ips = [local.mich_ip] } resource "cloudflare_r2_bucket" "static" { diff --git a/tf/deployment/modules/shared/cloudflare/account/terragrunt.hcl b/tf/deployment/modules/shared/cloudflare/account/terragrunt.hcl index 438d839c..ca18b9af 100644 --- a/tf/deployment/modules/shared/cloudflare/account/terragrunt.hcl +++ b/tf/deployment/modules/shared/cloudflare/account/terragrunt.hcl @@ -1,5 +1,5 @@ terraform { - source = "." + source = "../../../../../" extra_arguments custom_vars { commands = get_terraform_commands_that_need_vars() diff --git a/tf/shared/modules/cloudflare-pages-project/config.tf b/tf/shared/modules/cloudflare-pages-project/config.tf index 434447f2..5708025c 100644 --- a/tf/shared/modules/cloudflare-pages-project/config.tf +++ b/tf/shared/modules/cloudflare-pages-project/config.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudflare = { - source = "cloudflare/cloudflare" + source = "cloudflare/cloudflare" version = "~>4.46" } } diff --git a/tf/shared/modules/cloudflare-pages-project/locals.tf b/tf/shared/modules/cloudflare-pages-project/locals.tf index 6374de01..d8e86109 100644 --- a/tf/shared/modules/cloudflare-pages-project/locals.tf +++ b/tf/shared/modules/cloudflare-pages-project/locals.tf @@ -1,5 +1,5 @@ locals { - app_name = replace(var.app_name, "/[^a-zA-Z\\d]/", "-") - dashed_domain = replace(var.domain, "/[^a-zA-Z\\d]/", "-") + app_name = replace(var.app_name, "/[^a-zA-Z\\d]/", "-") + dashed_domain = replace(var.domain, "/[^a-zA-Z\\d]/", "-") sanitised_project_name = "${local.app_name}-${local.dashed_domain}-${var.env}" } diff --git a/tf/shared/modules/cloudflare-pages/config.tf b/tf/shared/modules/cloudflare-pages/config.tf index 434447f2..5708025c 100644 --- a/tf/shared/modules/cloudflare-pages/config.tf +++ b/tf/shared/modules/cloudflare-pages/config.tf @@ -3,7 +3,7 @@ terraform { required_providers { cloudflare = { - source = "cloudflare/cloudflare" + source = "cloudflare/cloudflare" version = "~>4.46" } } diff --git a/tf/shared/modules/cloudflare-pages/domain.tf b/tf/shared/modules/cloudflare-pages/domain.tf index 2dbf35ba..81abd133 100644 --- a/tf/shared/modules/cloudflare-pages/domain.tf +++ b/tf/shared/modules/cloudflare-pages/domain.tf @@ -2,8 +2,8 @@ module "domain" { source = "../domain" app_name = var.app_name - stage = var.stage - env = var.env + stage = var.stage + env = var.env } data "cloudflare_zone" "domain" { diff --git a/tf/shared/modules/cloudflare-pages/locals.tf b/tf/shared/modules/cloudflare-pages/locals.tf index 2ea15a0f..a9cc1630 100644 --- a/tf/shared/modules/cloudflare-pages/locals.tf +++ b/tf/shared/modules/cloudflare-pages/locals.tf @@ -3,7 +3,7 @@ locals { // This determines whether the deployment is production or staging // In our case we deploy to the "prod" production branch only for production environment with no stage // This automatically resolves if we combine stage and env - unsanitised_pages_branch = "${var.stage}${var.env}" - pages_branch = replace(local.unsanitised_pages_branch, "/[^a-zA-Z\\d]/", "-") - pages_url_prefix = local.pages_branch == "prod" ? "" : "${local.pages_branch}." + unsanitised_pages_branch = "${var.stage}${var.env}" + pages_branch = replace(local.unsanitised_pages_branch, "/[^a-zA-Z\\d]/", "-") + pages_url_prefix = local.pages_branch == "prod" ? "" : "${local.pages_branch}." } diff --git a/tf/shared/modules/domain/outputs.tf b/tf/shared/modules/domain/outputs.tf index e055e8fe..cdc64006 100644 --- a/tf/shared/modules/domain/outputs.tf +++ b/tf/shared/modules/domain/outputs.tf @@ -1,14 +1,14 @@ locals { // Only include stage name in domain if its set - domain_stage = var.stage == "" ? "" : "${var.stage}." + domain_stage = var.stage == "" ? "" : "${var.stage}." // We don't include the environment name in the URL for prod - domain_env = var.env == "prod" ? "" : "${var.env}." + domain_env = var.env == "prod" ? "" : "${var.env}." // Combine domain stage and environment, if stage is blank and env is prod, this will be an empty string domain_prefix = "${local.domain_stage}${local.domain_env}" // Example: buy.immich.app or buy.dev.immich.app or buy.pr-55.dev.immich.app - fqdn = "${var.app_name}.${local.domain_prefix}${var.domain}" + fqdn = "${var.app_name}.${local.domain_prefix}${var.domain}" } -output fqdn { +output "fqdn" { value = local.fqdn }