mirror of
https://github.com/immich-app/yucca.git
synced 2026-09-30 13:33:00 +08:00
feat(ceph): add sietch staging ansible inventory (#166)
This commit is contained in:
Andy Molenda
@@ -0,0 +1,111 @@
|
||||
---
|
||||
# === Naming ===
|
||||
cluster_name: sietch
|
||||
cluster_role: ceph
|
||||
|
||||
# === Network ===
|
||||
cluster_domain: staging.austin.int.futo.cloud
|
||||
public_network: 10.10.10.0/24
|
||||
cluster_network: 10.10.10.0/24
|
||||
gateway: 10.10.10.1
|
||||
dns_server: 10.10.10.1
|
||||
bond_mode: active-backup
|
||||
bond_interfaces:
|
||||
- eno1np0
|
||||
- eno2np1
|
||||
networkd_enabled: true
|
||||
|
||||
# === Ceph ===
|
||||
ceph_release: tentacle
|
||||
ceph_repo_url: "https://download.ceph.com/debian-{{ ceph_release }}/"
|
||||
ceph_repo_key_url: "https://download.ceph.com/keys/release.asc"
|
||||
|
||||
# === OS Provisioning ===
|
||||
admin_user: ansible-iac
|
||||
timezone: UTC
|
||||
|
||||
# Deploy keypair path on the controller. Both {path} and {path}.pub must
|
||||
# exist before running provision.yml (the preflight play in provision.yml
|
||||
# enforces this). ansible-iac's authorized_keys on every node is populated
|
||||
# by a file lookup at provision time — rotating the key on the controller
|
||||
# automatically propagates on the next re-provision.
|
||||
provision_iac_ssh_key_path: "~/.ssh/id_ed25519_sietch"
|
||||
|
||||
# 1P vault for cluster secret lookups (e.g., rotate-ssh-key.yml reads pubkey from here).
|
||||
cluster_secrets_vault: yucca_tf_staging
|
||||
|
||||
# Secret aliases — vault_* vars populated by scripts/ansible-play.sh via op inject
|
||||
#
|
||||
# Note: the deploy public key is NOT a secret — it's public data and lives
|
||||
# at {{ provision_iac_ssh_key_path }}.pub on the controller. Reading it via
|
||||
# file lookup avoids drift between vault and disk.
|
||||
ops_password: "{{ vault_ops_password }}"
|
||||
ceph_dashboard_user: admin
|
||||
ceph_dashboard_password: "{{ vault_ceph_dashboard_password }}"
|
||||
|
||||
# S3 svc-user (yucca-restic consumer) — TF+1P-predetermined keys passed to
|
||||
# `radosgw-admin user create --access-key=... --secret-key=...` so the Yucca
|
||||
# app can be pre-configured with matching credentials. Rotation path documented
|
||||
# in docs/runbooks/rotate-secrets.md.
|
||||
ceph_rgw_s3_user_access_key: "{{ vault_s3_restic_access_key }}"
|
||||
ceph_rgw_s3_user_secret_key: "{{ vault_s3_restic_secret_key }}"
|
||||
|
||||
# === Storage ===
|
||||
ssd_model_pattern: "Micron_5100"
|
||||
|
||||
os_partitions:
|
||||
esp_size: "512M"
|
||||
boot_size: "1G"
|
||||
root_size: "80G"
|
||||
swap_size: "8G"
|
||||
ceph_db_size: "1440G"
|
||||
|
||||
ceph_db_lv_size: "240G"
|
||||
ceph_db_lvs_per_ssd: 6
|
||||
|
||||
# === RGW (Object Gateway) ===
|
||||
ceph_rgw_realm: sietch
|
||||
ceph_rgw_zonegroup: us-east-1
|
||||
ceph_rgw_zonegroup_api_name: us-east-1
|
||||
ceph_rgw_zone: staging-z1
|
||||
ceph_rgw_ec_profile: ec-k8m3-osd
|
||||
ceph_rgw_ec_k: 8
|
||||
ceph_rgw_ec_m: 3
|
||||
ceph_rgw_ec_failure_domain: osd
|
||||
ceph_rgw_ec_device_class: hdd
|
||||
ceph_rgw_data_pool: "{{ ceph_rgw_zone }}.rgw.buckets.data"
|
||||
ceph_rgw_index_pool: "{{ ceph_rgw_zone }}.rgw.buckets.index"
|
||||
ceph_rgw_extra_pool: "{{ ceph_rgw_zone }}.rgw.buckets.non-ec"
|
||||
ceph_rgw_replicated_size: 2
|
||||
ceph_rgw_replicated_min_size: 1
|
||||
ceph_rgw_port: 443
|
||||
ceph_rgw_s3_user_uid: svc-yucca-restic
|
||||
ceph_rgw_s3_user_display_name: "yucca/restic service account"
|
||||
|
||||
# --- RGW DNS + TLS ---
|
||||
# Virtual-hosted S3 support: setting rgw_dns_name tells RGW to strip this
|
||||
# suffix from the Host header and treat the remainder as the bucket name.
|
||||
# Requires matching DNS: both s3.<domain> and *.s3.<domain> should resolve
|
||||
# to the cluster nodes (round-robin A, or VIP/LB in prod).
|
||||
ceph_rgw_dns_name: s3.staging.austin.int.futo.cloud
|
||||
|
||||
# Self-signed wildcard cert handed to cephadm via service spec.
|
||||
# cephadm distributes to all RGW daemons. To rotate, delete
|
||||
# /etc/ceph/rgw-ssl.{crt,key} on the bootstrap node and re-run the role.
|
||||
ceph_rgw_ssl: true
|
||||
ceph_rgw_ssl_cert_days: 3650 # 10 years
|
||||
ceph_rgw_ssl_cert_subject_c: US
|
||||
ceph_rgw_ssl_cert_subject_st: Texas
|
||||
ceph_rgw_ssl_cert_subject_l: Austin
|
||||
ceph_rgw_ssl_cert_subject_o: FUTO
|
||||
ceph_rgw_ssl_cert_email: yucca@futo.org
|
||||
|
||||
# Computed: scheme used for endpoints, debug output, and zonegroup/zone URLs
|
||||
ceph_rgw_scheme: "{{ 'https' if ceph_rgw_ssl else 'http' }}"
|
||||
|
||||
# === Monitoring Stack ===
|
||||
ceph_prometheus_port: 9095
|
||||
ceph_grafana_port: 3000
|
||||
ceph_alertmanager_port: 9093
|
||||
ceph_grafana_admin_user: admin
|
||||
ceph_grafana_admin_password: "{{ vault_grafana_admin_password }}"
|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
# Copy to host_vars/sietch-ceph-<name>.yml (full inventory_hostname).
|
||||
# The <name> segment is either operator-declared in clusters.auto.tfvars
|
||||
# or TF-auto-picked from wordlist.txt — see docs/naming.md.
|
||||
# Filename MUST match inventory_hostname for Ansible auto-load.
|
||||
# Values are node-specific — hardware paths differ per chassis.
|
||||
hostname_short: sietch-ceph-EXAMPLE
|
||||
bond_ip: 10.0.0.1
|
||||
|
||||
# SAS expander base path (unique per chassis)
|
||||
# Find with: ls /dev/disk/by-path/ | grep sas
|
||||
sas_path_prefix: "pci-0000:02:00.0-sas-exp0x500056b3XXXXXXXX"
|
||||
|
||||
# SSD PHY positions in the SAS topology
|
||||
ssd1_phy: 12
|
||||
ssd2_phy: 13
|
||||
|
||||
# LVM volume group names for block.db (on SSD partition 5)
|
||||
ceph_db_vg1: ceph-db-ssd1
|
||||
ceph_db_vg2: ceph-db-ssd2
|
||||
|
||||
# HDD OSD mappings: SAS PHY slot -> block.db LV
|
||||
# 6 HDDs per SSD, each gets a dedicated 240G block.db LV
|
||||
ceph_hdd_osds:
|
||||
- path_phy: phy0
|
||||
db: ceph-db-ssd1/db-slot0
|
||||
- path_phy: phy1
|
||||
db: ceph-db-ssd1/db-slot1
|
||||
# ... one entry per HDD
|
||||
|
||||
# SSD OSD partitions (partition 6 on each SSD, no separate block.db)
|
||||
ceph_ssd_osds:
|
||||
- path_phy: phy12
|
||||
partition: 6
|
||||
- path_phy: phy13
|
||||
partition: 6
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
---
|
||||
hostname_short: sietch-ceph-laurel
|
||||
bond_ip: 10.10.10.90
|
||||
|
||||
# SAS expander base path (unique per chassis — different backplane address per node)
|
||||
sas_path_prefix: "pci-0000:02:00.0-sas-exp0x500056b3fcf498ff"
|
||||
|
||||
# SSD PHY positions (rear bays)
|
||||
ssd1_phy: 12 # serial 17321A07BA4A
|
||||
ssd2_phy: 13 # serial 17321A07CFEE
|
||||
|
||||
# LVM VGs on SSD partition 5
|
||||
ceph_db_vg1: ceph-db-rear12 # VG on SSD1 (phy12) partition 5
|
||||
ceph_db_vg2: ceph-db-rear13 # VG on SSD2 (phy13) partition 5
|
||||
|
||||
# HDD OSD mappings: PHY slot -> block.db LV
|
||||
# PHY 0-5 -> SSD1 (ceph-db-rear12/db-slot0..5)
|
||||
# PHY 6-11 -> SSD2 (ceph-db-rear13/db-slot6..11)
|
||||
# by-path is slot-stable: replacing a drive in the same bay keeps the same path
|
||||
# 12 HDDs — all front bays populated
|
||||
ceph_hdd_osds:
|
||||
- path_phy: phy0
|
||||
db: ceph-db-rear12/db-slot0
|
||||
- path_phy: phy1
|
||||
db: ceph-db-rear12/db-slot1
|
||||
- path_phy: phy2
|
||||
db: ceph-db-rear12/db-slot2
|
||||
- path_phy: phy3
|
||||
db: ceph-db-rear12/db-slot3 # serial Z4D09B99 (ST6000NKCLAR6000, added 2026-04-10)
|
||||
- path_phy: phy4
|
||||
db: ceph-db-rear12/db-slot4
|
||||
- path_phy: phy5
|
||||
db: ceph-db-rear12/db-slot5 # serial Z4D0G7VC (ST6000NKCLAR6000, added 2026-04-10)
|
||||
- path_phy: phy6
|
||||
db: ceph-db-rear13/db-slot6
|
||||
- path_phy: phy7
|
||||
db: ceph-db-rear13/db-slot7 # serial Z4D0G7SC (ST6000NKCLAR6000, added 2026-04-10)
|
||||
- path_phy: phy8
|
||||
db: ceph-db-rear13/db-slot8
|
||||
- path_phy: phy9
|
||||
db: ceph-db-rear13/db-slot9
|
||||
- path_phy: phy10
|
||||
db: ceph-db-rear13/db-slot10 # serial Z4D0G7XK (ST6000NKCLAR6000, added 2026-04-10)
|
||||
- path_phy: phy11
|
||||
db: ceph-db-rear13/db-slot11
|
||||
|
||||
# SSD OSD partitions (partition 6 on each rear SSD, no separate block.db)
|
||||
ceph_ssd_osds:
|
||||
- path_phy: phy12
|
||||
partition: 6
|
||||
- path_phy: phy13
|
||||
partition: 6
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
---
|
||||
hostname_short: sietch-ceph-lawson
|
||||
bond_ip: 10.10.10.91
|
||||
|
||||
# SAS expander base path (unique per chassis — different backplane address per node)
|
||||
sas_path_prefix: "pci-0000:02:00.0-sas-exp0x500056b35be6b4ff"
|
||||
|
||||
# SSD PHY positions (rear bays)
|
||||
ssd1_phy: 12 # serial 17321A07CF91
|
||||
ssd2_phy: 13 # serial 17251B44C4D8
|
||||
|
||||
# LVM VGs on SSD partition 5
|
||||
ceph_db_vg1: ceph-db-rear12 # VG on SSD1 (phy12) partition 5
|
||||
ceph_db_vg2: ceph-db-rear13 # VG on SSD2 (phy13) partition 5
|
||||
|
||||
# HDD OSD mappings: PHY slot -> block.db LV
|
||||
# PHY 0-5 -> SSD1 (ceph-db-rear12/db-slot0..5)
|
||||
# PHY 6-11 -> SSD2 (ceph-db-rear13/db-slot6..11)
|
||||
# by-path is slot-stable: replacing a drive in the same bay keeps the same path
|
||||
# 12 HDDs — all front bays populated
|
||||
ceph_hdd_osds:
|
||||
- path_phy: phy0
|
||||
db: ceph-db-rear12/db-slot0
|
||||
- path_phy: phy1
|
||||
db: ceph-db-rear12/db-slot1
|
||||
- path_phy: phy2
|
||||
db: ceph-db-rear12/db-slot2
|
||||
- path_phy: phy3
|
||||
db: ceph-db-rear12/db-slot3
|
||||
- path_phy: phy4
|
||||
db: ceph-db-rear12/db-slot4
|
||||
- path_phy: phy5
|
||||
db: ceph-db-rear12/db-slot5
|
||||
- path_phy: phy6
|
||||
db: ceph-db-rear13/db-slot6
|
||||
- path_phy: phy7
|
||||
db: ceph-db-rear13/db-slot7 # serial Z4D0GKJX (ST6000NKCLAR6000, added 2026-04-10)
|
||||
- path_phy: phy8
|
||||
db: ceph-db-rear13/db-slot8
|
||||
- path_phy: phy9
|
||||
db: ceph-db-rear13/db-slot9
|
||||
- path_phy: phy10
|
||||
db: ceph-db-rear13/db-slot10
|
||||
- path_phy: phy11
|
||||
db: ceph-db-rear13/db-slot11
|
||||
|
||||
# SSD OSD partitions (partition 6 on each rear SSD, no separate block.db)
|
||||
ceph_ssd_osds:
|
||||
- path_phy: phy12
|
||||
partition: 6
|
||||
- path_phy: phy13
|
||||
partition: 6
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
---
|
||||
hostname_short: sietch-ceph-samara
|
||||
bond_ip: 10.10.10.92
|
||||
|
||||
# SAS expander base path (unique per chassis — different backplane address per node)
|
||||
sas_path_prefix: "pci-0000:02:00.0-sas-exp0x500056b3393ba1ff"
|
||||
|
||||
# SSD PHY positions (rear bays)
|
||||
ssd1_phy: 12 # serial 17321A07D1D3
|
||||
ssd2_phy: 13 # serial 17251B44DF51
|
||||
|
||||
# LVM VGs on SSD partition 5
|
||||
ceph_db_vg1: ceph-db-rear12 # VG on SSD1 (phy12) partition 5
|
||||
ceph_db_vg2: ceph-db-rear13 # VG on SSD2 (phy13) partition 5
|
||||
|
||||
# HDD OSD mappings: PHY slot -> block.db LV
|
||||
# PHY 0-5 -> SSD1 (ceph-db-rear12/db-slot0..5)
|
||||
# PHY 6-11 -> SSD2 (ceph-db-rear13/db-slot6..11)
|
||||
# by-path is slot-stable: replacing a drive in the same bay keeps the same path
|
||||
# 12 HDDs — all front bays populated
|
||||
ceph_hdd_osds:
|
||||
- path_phy: phy0
|
||||
db: ceph-db-rear12/db-slot0
|
||||
- path_phy: phy1
|
||||
db: ceph-db-rear12/db-slot1
|
||||
- path_phy: phy2
|
||||
db: ceph-db-rear12/db-slot2
|
||||
- path_phy: phy3
|
||||
db: ceph-db-rear12/db-slot3
|
||||
- path_phy: phy4
|
||||
db: ceph-db-rear12/db-slot4 # serial Z4D0GKB2 (replacement, added 2026-04-11)
|
||||
- path_phy: phy5
|
||||
db: ceph-db-rear12/db-slot5
|
||||
- path_phy: phy6
|
||||
db: ceph-db-rear13/db-slot6
|
||||
- path_phy: phy7
|
||||
db: ceph-db-rear13/db-slot7
|
||||
- path_phy: phy8
|
||||
db: ceph-db-rear13/db-slot8
|
||||
- path_phy: phy9
|
||||
db: ceph-db-rear13/db-slot9
|
||||
- path_phy: phy10
|
||||
db: ceph-db-rear13/db-slot10
|
||||
- path_phy: phy11
|
||||
db: ceph-db-rear13/db-slot11
|
||||
|
||||
# SSD OSD partitions (partition 6 on each rear SSD, no separate block.db)
|
||||
ceph_ssd_osds:
|
||||
- path_phy: phy12
|
||||
partition: 6
|
||||
- path_phy: phy13
|
||||
partition: 6
|
||||
Reference in New Issue
Block a user