fix(ceph): pin SATA link power to max_performance on OSD nodes (#347)

* fix(ceph): pin SATA link power to max_performance on OSD nodes

* feat(ceph): drift-check the SATA link power management policy

ansible/ceph/drift.yml

@@ -1,5 +1,5 @@
 ---
-# Drift detection — compares expected Ansible state against live cluster.
+# Drift detection: compares expected Ansible state against live cluster.
 # Read-only. No changes. Reports mismatches.
 #
 # Usage:
@@ -113,6 +113,44 @@
                      == ceph_ssd_scheduler
           }] }}
 
+    # SATA link power management
+    # Every AHCI port matters here, not a representative one: a single port
+    # back on min_power_with_partial is enough to stall an OSD, so report the
+    # distribution when the ports disagree.
+    - name: Check SATA link power management policy
+      ansible.builtin.shell: |
+        set -o pipefail
+        GLOB=/sys/class/scsi_host/host*/link_power_management_policy
+        DISTINCT=$(cat $GLOB 2>/dev/null | sort -u | grep -c .)
+        if [ "$DISTINCT" -eq 0 ]; then
+          # No LPM-capable ports (SAS controllers expose scsi_host without the
+          # attribute). The role skips these too, so there is nothing to drift.
+          echo n/a
+        elif [ "$DISTINCT" -eq 1 ]; then
+          cat $GLOB 2>/dev/null | sort -u
+        else
+          cat $GLOB 2>/dev/null | sort | uniq -c \
+            | awk '{printf "%s=%s ", $2, $1}' | sed 's/^/mixed: /; s/ $//'
+        fi
+      args:
+        executable: /bin/bash
+      register: sata_lpm
+      changed_when: false
+      when: ceph_sata_link_pm_enabled | bool
+
+    - name: Record SATA link power management drift
+      ansible.builtin.set_fact:
+        drift_results: >-
+          {{ drift_results + [{
+            'category': 'hardware',
+            'item': 'SATA link power mgmt',
+            'expected': ceph_sata_link_pm_policy,
+            'actual': sata_lpm.stdout | trim | default('n/a'),
+            'match': (sata_lpm.stdout | trim | default('n/a'))
+                     in [ceph_sata_link_pm_policy, 'n/a']
+          }] }}
+      when: ceph_sata_link_pm_enabled | bool
+
     # nftables
     - name: Check nftables policy
       ansible.builtin.shell: |

ansible/ceph/roles/hardware_tuning/defaults/main.yml

@@ -11,7 +11,7 @@ ceph_ssd_scheduler: none
 ceph_hdd_readahead_kb: 4096     # 4 MB
 ceph_ssd_readahead_kb: 128      # 128 KB (kernel default)
 
-# nr_requests (disk queue depth) — 0 means leave at kernel default
+# nr_requests (disk queue depth); 0 means leave at kernel default
 ceph_hdd_nr_requests: 128
 ceph_ssd_nr_requests: 0         # NVMe/SSD defaults are fine
 
@@ -19,6 +19,22 @@ ceph_ssd_nr_requests: 0         # NVMe/SSD defaults are fine
 # BlueStore manages discard for OSD devices internally.
 ceph_enable_fstrim_timer: true
 
+# SATA link power management (ALPM).
+#
+# The AMD FCH AHCI ports come up as min_power_with_partial, which lets the SATA
+# PHY drop into Partial/Slumber between I/Os. Waking it costs milliseconds, and
+# on 2026-07-24 that surfaced as a BlueStore slow-op alert on osd.155 (curtis
+# sdk): PHYRdyChg/CommWake in dmesg, drive itself healthy.
+#
+# An OSD HDD is never idle enough for the power saving to be worth the wake
+# latency, so pin the link to max_performance. Applied live and persisted via
+# the same udev rules file as the block-device tuning.
+#
+# NOTE: writing this attribute schedules a libata EH cycle on the port, so roll
+# it out per node (serial: 1) rather than fleet-wide in one pass.
+ceph_sata_link_pm_enabled: true
+ceph_sata_link_pm_policy: max_performance
+
 # Transparent Huge Pages -- madvise is universally correct for BlueStore.
 # THP=always drives tcmalloc/BlueStore RSS bloat and allocation-stall latency.
 # Applied on all ceph clusters (not gated behind the CPU governor).
@@ -26,13 +42,13 @@ ceph_thp_enabled: true
 ceph_thp_mode: madvise          # /sys/kernel/mm/transparent_hugepage/enabled
 ceph_thp_defrag: madvise        # /sys/kernel/mm/transparent_hugepage/defrag
 
-# CPU governor — force performance mode (disables power saving C-states)
+# CPU governor: force performance mode (disables power saving C-states)
 # Useful for dedicated Ceph nodes where latency matters more than power.
 ceph_cpu_governor_enabled: false
 ceph_cpu_governor: performance
 ceph_cpu_max_cstate: 1    # kernel param processor.max_cstate
 
-# LVM device filter — restrict LVM scanning to known VGs.
+# LVM device filter: restrict LVM scanning to known VGs.
 # Prevents LVM from scanning raw OSD disks (speeds up lvscan/pvscan).
 ceph_lvm_filter_enabled: false
 ceph_lvm_filter: '[ "a|/dev/vg0/.*|", "a|/dev/md.*|", "r|.*|" ]'

ansible/ceph/roles/hardware_tuning/tasks/main.yml

@@ -77,6 +77,30 @@
     state: started
   when: ceph_enable_fstrim_timer | bool
 
+# ---------- SATA link power management ----------
+
+# The udev rule above covers boot and hot-plug; this applies the policy to
+# ports that are already up. Writing the attribute schedules a libata EH cycle
+# on that port, which is why this role wants serial: 1 on a serving cluster.
+- name: Set SATA link power management policy live
+  ansible.builtin.shell: |
+    set -uo pipefail
+    POLICY="{{ ceph_sata_link_pm_policy }}"
+    CHANGED=0
+    for f in /sys/class/scsi_host/host*/link_power_management_policy; do
+      [ -w "$f" ] || continue
+      CUR=$(cat "$f" 2>/dev/null || echo "")
+      if [ "$CUR" != "$POLICY" ]; then
+        echo "$POLICY" > "$f" 2>/dev/null && CHANGED=1 || true
+      fi
+    done
+    [ "$CHANGED" -eq 0 ] || echo CHANGED
+  args:
+    executable: /bin/bash
+  register: _sata_lpm
+  changed_when: "'CHANGED' in (_sata_lpm.stdout | default(''))"
+  when: ceph_sata_link_pm_enabled | bool
+
 # ---------- Transparent Huge Pages ----------
 
 - name: Set Transparent Huge Pages live
@@ -207,6 +231,22 @@
   register: scheduler_verify
   changed_when: false
 
+- name: Verify SATA link power management policy
+  ansible.builtin.shell: |
+    set -uo pipefail
+    echo "=== SATA link power management ==="
+    for f in /sys/class/scsi_host/host*/link_power_management_policy; do
+      [ -e "$f" ] || continue
+      cat "$f"
+    done | sort | uniq -c | sed 's/^/  /'
+  args:
+    executable: /bin/bash
+  register: sata_lpm_verify
+  changed_when: false
+  when: ceph_sata_link_pm_enabled | bool
+
 - name: Display disk tuning state
   ansible.builtin.debug:
-    msg: "{{ scheduler_verify.stdout_lines }}"
+    msg: >-
+      {{ scheduler_verify.stdout_lines
+         + (sata_lpm_verify.stdout_lines | default([])) }}

ansible/ceph/roles/hardware_tuning/templates/99-ceph-disk-tuning.rules.j2

@@ -1,4 +1,4 @@
-# Ceph disk tuning udev rules — managed by Ansible (hardware-tuning role)
+# Ceph disk tuning udev rules, managed by Ansible (hardware-tuning role)
 # Applied on device add/change events. Survives reboots and hot-plug.
 
 # HDD: rotational devices
@@ -14,3 +14,11 @@ ACTION=="add|change", SUBSYSTEM=="block", ATTR{queue/rotational}=="0", ATTR{queu
 {% if ceph_ssd_nr_requests > 0 %}
 ACTION=="add|change", SUBSYSTEM=="block", ATTR{queue/rotational}=="0", ATTR{queue/nr_requests}="{{ ceph_ssd_nr_requests }}"
 {% endif %}
+{% if ceph_sata_link_pm_enabled %}
+
+# SATA link power management: keep the PHY out of Partial/Slumber. Waking it
+# costs milliseconds and shows up as a BlueStore slow op, and an OSD disk is
+# never idle long enough for the saving to be worth it.
+# TEST== guards non-SATA hosts, which have no such attribute.
+ACTION=="add|change", SUBSYSTEM=="scsi_host", KERNEL=="host*", TEST=="link_power_management_policy", ATTR{link_power_management_policy}="{{ ceph_sata_link_pm_policy }}"
+{% endif %}