mirror of
https://github.com/immich-app/yucca.git
synced 2026-09-30 13:33:00 +08:00
fix(all): adjust benchmark related configs and fix metrics (#332)
This commit is contained in:
Antoine Lecompte
@@ -23,8 +23,10 @@ OP_SERVICE_ACCOUNT_TOKEN=$(op read "${ACCT[@]}" \
|
||||
export OP_SERVICE_ACCOUNT_TOKEN
|
||||
|
||||
# Render the NETCONF SSH key from 1Password to a 0600 temp file (op run can't
|
||||
# write files) — now via the escalated SA.
|
||||
KEYF=$(mktemp); chmod 600 "$KEYF"; trap 'rm -f "$KEYF"' EXIT
|
||||
# write files) — now via the escalated SA. DETERMINISTIC path: must match the
|
||||
# plan job's render exactly (the value lives inside the saved plan).
|
||||
KEYF="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/netconf-terraform-key"
|
||||
rm -f "$KEYF"; ( umask 077; : > "$KEYF" ); trap 'rm -f "$KEYF"' EXIT
|
||||
op read "op://yucca_tf_prod/NET_SWITCHES_TERRAFORM_SSH_PRIVATE_KEY/password" > "$KEYF"
|
||||
|
||||
export TF_VAR_netconf_ssh_key_path="$KEYF"
|
||||
|
||||
@@ -23,8 +23,11 @@ if [ -z "${OP_SERVICE_ACCOUNT_TOKEN:-}" ]; then
|
||||
fi
|
||||
|
||||
# Render the NETCONF SSH key from 1Password to a 0600 temp file (op run can't
|
||||
# write files).
|
||||
KEYF=$(mktemp); chmod 600 "$KEYF"; trap 'rm -f "$KEYF"' EXIT
|
||||
# write files). DETERMINISTIC path: the value is recorded in the saved plan, so
|
||||
# the gated apply (a different runner) must render to the identical path or
|
||||
# tofu rejects the plan ("Mismatch between input and plan variable value").
|
||||
KEYF="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/netconf-terraform-key"
|
||||
rm -f "$KEYF"; ( umask 077; : > "$KEYF" ); trap 'rm -f "$KEYF"' EXIT
|
||||
op read "op://yucca_tf_prod/NET_SWITCHES_TERRAFORM_SSH_PRIVATE_KEY/password" > "$KEYF"
|
||||
|
||||
export TF_VAR_netconf_ssh_key_path="$KEYF"
|
||||
|
||||
@@ -49,8 +49,28 @@ spec:
|
||||
serviceSelector:
|
||||
matchLabels:
|
||||
lb: internal
|
||||
# Ranges, not the full /24: .17 is carved out for lb-gw-internal below
|
||||
# (overlapping pools would mark each other conflicting and disable).
|
||||
blocks:
|
||||
- cidr: 10.40.12.0/24
|
||||
- start: 10.40.12.1
|
||||
stop: 10.40.12.16
|
||||
- start: 10.40.12.18
|
||||
stop: 10.40.12.254
|
||||
---
|
||||
# Dedicated internal VIP for the gw (michael) envoy Service: it already draws
|
||||
# its public IP from pool-a (lb: public), so a second, label-disjoint pool
|
||||
# grants the internal twin (${GW_INT_VIP}) requested via lbipam.cilium.io/ips.
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumLoadBalancerIPPool
|
||||
metadata:
|
||||
name: lb-gw-internal
|
||||
spec:
|
||||
serviceSelector:
|
||||
matchLabels:
|
||||
lb-internal-vip: gw
|
||||
blocks:
|
||||
- start: 10.40.12.17
|
||||
stop: 10.40.12.17
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumBGPAdvertisement
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
# Internal (on-net) access to michael via ${GW_INT_HOST} → ${GW_INT_VIP}.
|
||||
# The public gw VIP rides the transit aggregate and is NOT routable from inside
|
||||
# Hetzner (mgmt hosts); this twin listener/VIP serves the same michael backend
|
||||
# over the fabric. DNS lives in the NetBird zone (netbird stack dns.tf `gw`).
|
||||
#
|
||||
# Cert: same DNS-01 pipeline as netops-wildcard; the Secret must live beside
|
||||
# the Gateway (envoy-system).
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: gw-internal
|
||||
namespace: envoy-system
|
||||
spec:
|
||||
secretName: gw-internal-tls
|
||||
issuerRef:
|
||||
name: letsencrypt-production
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- "${GW_INT_HOST}"
|
||||
---
|
||||
# Route the internal hostname to michael — mirror of apps/base/httproutes/gw.yaml
|
||||
# (which stays ${GW_HOST}-only; this listener is prod-specific).
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: gw-internal
|
||||
namespace: yucca
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: gw
|
||||
namespace: envoy-system
|
||||
sectionName: https-internal
|
||||
hostnames:
|
||||
- "${GW_INT_HOST}"
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: yucca-michael
|
||||
port: 3010
|
||||
@@ -36,8 +36,10 @@ spec:
|
||||
metadata:
|
||||
labels:
|
||||
lb: public
|
||||
# Second, internal VIP (lb-gw-internal pool) for on-net clients.
|
||||
lb-internal-vip: gw
|
||||
annotations:
|
||||
lbipam.cilium.io/ips: "${GW_VIP}"
|
||||
lbipam.cilium.io/ips: "${GW_VIP},${GW_INT_VIP}"
|
||||
telemetry:
|
||||
metrics:
|
||||
prometheus: {}
|
||||
|
||||
@@ -28,3 +28,17 @@ spec:
|
||||
certificateRefs:
|
||||
- kind: Secret
|
||||
name: app-domain-tls
|
||||
# Internal twin (${GW_INT_VIP}): same envoy fleet, NetBird-zone hostname,
|
||||
# own DNS-01 cert (gw-internal.yaml at the overlay root).
|
||||
- name: https-internal
|
||||
protocol: HTTPS
|
||||
port: 443
|
||||
hostname: "${GW_INT_HOST}"
|
||||
allowedRoutes:
|
||||
namespaces:
|
||||
from: All
|
||||
tls:
|
||||
mode: Terminate
|
||||
certificateRefs:
|
||||
- kind: Secret
|
||||
name: gw-internal-tls
|
||||
|
||||
@@ -16,6 +16,7 @@ resources:
|
||||
- ./flux-system
|
||||
- ./coredns.yaml
|
||||
- ./cilium-bgp.yaml
|
||||
- ./gw-internal.yaml
|
||||
- ./lb-return-route.yaml
|
||||
- ./diskpools.yaml
|
||||
- ./netops
|
||||
|
||||
@@ -31,6 +31,11 @@ data:
|
||||
# externalTrafficPolicy: Local + one envoy per worker → spine ECMP.
|
||||
GW_PARENT_GATEWAY: gw
|
||||
GW_VIP: 69.48.224.6
|
||||
# Internal (NetBird-zone) twin of the gw VIP: on-net clients (mgmt hosts,
|
||||
# yuctl tools bench) can't reach the public aggregate from inside Hetzner.
|
||||
# DNS: netbird dns.tf `gw` record; pool: lb-gw-internal in cilium-bgp.yaml.
|
||||
GW_INT_VIP: 10.40.12.17
|
||||
GW_INT_HOST: gw.father.fsn.htz.yucca.futo.network
|
||||
GW_PROXY_REPLICAS: "3"
|
||||
# (netops keeps its own internal VIP, pinned at 10.40.12.16 in netops/.)
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "catalog:",
|
||||
"@opentelemetry/instrumentation-pino": "catalog:",
|
||||
"@opentelemetry/propagator-b3": "catalog:",
|
||||
"@opentelemetry/resources": "catalog:",
|
||||
"@opentelemetry/propagator-jaeger": "catalog:",
|
||||
"@opentelemetry/sdk-node": "catalog:",
|
||||
"nestjs-otel": "catalog:",
|
||||
|
||||
@@ -6,13 +6,20 @@ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino';
|
||||
import { B3Propagator } from '@opentelemetry/propagator-b3';
|
||||
import { JaegerPropagator } from '@opentelemetry/propagator-jaeger';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import { logs, metrics, NodeSDK, tracing } from '@opentelemetry/sdk-node';
|
||||
import { hostname } from 'node:os';
|
||||
import { otelEnv } from './env.js';
|
||||
|
||||
const SpanProcessor = otelEnv.NODE_ENV === 'development' ? tracing.SimpleSpanProcessor : tracing.BatchSpanProcessor;
|
||||
const LogProcessor = otelEnv.NODE_ENV === 'development' ? logs.SimpleLogRecordProcessor : logs.BatchLogRecordProcessor;
|
||||
|
||||
const otelSDK = new NodeSDK({
|
||||
// Without service.instance.id every replica exports IDENTICAL series; the
|
||||
// TSDB merges them and rate() reads ~1/replicas of the real traffic. The SDK
|
||||
// merges this with its detected resource (service.name via OTEL_SERVICE_NAME).
|
||||
resource: resourceFromAttributes({ 'service.instance.id': hostname() }),
|
||||
|
||||
// metrics
|
||||
metricReader: new metrics.PeriodicExportingMetricReader({
|
||||
exporter: new OTLPMetricExporter({
|
||||
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.41.0
|
||||
go.opentelemetry.io/otel/log v0.18.0
|
||||
go.opentelemetry.io/otel/metric v1.42.0
|
||||
go.opentelemetry.io/otel/sdk v1.42.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.18.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0
|
||||
)
|
||||
@@ -38,7 +39,6 @@ require (
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
|
||||
@@ -1,47 +1,25 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=
|
||||
github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
|
||||
@@ -5,20 +5,40 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"michael/internal/auth"
|
||||
"michael/internal/config"
|
||||
"michael/internal/version"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
otelmetric "go.opentelemetry.io/otel/metric"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
sdkresource "go.opentelemetry.io/otel/sdk/resource"
|
||||
)
|
||||
|
||||
// otelResource identifies this process to the collector. Without
|
||||
// service.instance.id every replica exports IDENTICAL series; the TSDB merges
|
||||
// them into one, interleaved cumulative counters read as resets, and rate()
|
||||
// reports ~1/replicas of the real traffic.
|
||||
func otelResource() *sdkresource.Resource {
|
||||
host, _ := os.Hostname()
|
||||
res, err := sdkresource.Merge(sdkresource.Default(), sdkresource.NewSchemaless(
|
||||
attribute.String("service.name", "michael"),
|
||||
attribute.String("service.version", version.Version),
|
||||
attribute.String("service.instance.id", host),
|
||||
))
|
||||
if err != nil {
|
||||
return sdkresource.Default()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
RequestedBytes otelmetric.Int64Counter
|
||||
DownloadedBytes otelmetric.Int64Counter
|
||||
@@ -102,6 +122,7 @@ func SetupMeterProvider(cfg config.Config) (*sdkmetric.MeterProvider, error) {
|
||||
}
|
||||
|
||||
provider := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithResource(otelResource()),
|
||||
sdkmetric.WithReader(
|
||||
sdkmetric.NewPeriodicReader(exporter,
|
||||
sdkmetric.WithInterval(cfg.OTLPMetricsInterval),
|
||||
|
||||
@@ -47,6 +47,7 @@ func SetupLogProvider(cfg config.Config) (*sdklog.LoggerProvider, error) {
|
||||
}
|
||||
|
||||
provider := sdklog.NewLoggerProvider(
|
||||
sdklog.WithResource(otelResource()),
|
||||
sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)),
|
||||
)
|
||||
return provider, nil
|
||||
|
||||
@@ -17,10 +17,11 @@ const remoteDir = ".cache/yuctl-bench/bin"
|
||||
|
||||
// RunOpts drives one remote benchmark run from the dev machine.
|
||||
type RunOpts struct {
|
||||
Host string // ssh destination for the management host
|
||||
AgentBin string // local linux/amd64 bench-agent; "" = use the embedded one
|
||||
Config Config
|
||||
Out string // local results path ("" = don't save)
|
||||
Host string // ssh destination for the management host
|
||||
SSHIdentity string // ssh private key file ("" = ssh defaults/agent)
|
||||
AgentBin string // local linux/amd64 bench-agent; "" = use the embedded one
|
||||
Config Config
|
||||
Out string // local results path ("" = don't save)
|
||||
}
|
||||
|
||||
// Run pushes the agent + pinned restic to the host, streams the run, and
|
||||
@@ -37,13 +38,13 @@ func Run(ctx context.Context, opts RunOpts) (*RunResult, error) {
|
||||
}
|
||||
|
||||
log.Info().Str("host", opts.Host).Msg("pushing agent + restic " + ResticVersion)
|
||||
if err := push(ctx, opts.Host, agentBin, resticBin); err != nil {
|
||||
if err := push(ctx, opts.SSHIdentity, opts.Host, agentBin, resticBin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Info().Str("host", opts.Host).Ints("connections", opts.Config.Connections).
|
||||
Str("size", FormatBytes(opts.Config.Size)).Msg("starting remote benchmark")
|
||||
result, err := drive(ctx, opts.Host, opts.Config)
|
||||
result, err := drive(ctx, opts.SSHIdentity, opts.Host, opts.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -83,13 +84,23 @@ func finish(result *RunResult, out string) (*RunResult, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func sshBase(host string) []string {
|
||||
return []string{
|
||||
// sshOpts builds the common ssh/scp options. accept-new (TOFU) keeps first
|
||||
// contact with a discovery-resolved IP from failing BatchMode.
|
||||
func sshOpts(identity string) []string {
|
||||
args := []string{
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ServerAliveInterval=30",
|
||||
"-o", "ServerAliveCountMax=8",
|
||||
host,
|
||||
}
|
||||
if identity != "" {
|
||||
args = append(args, "-i", identity, "-o", "IdentitiesOnly=yes")
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func sshBase(identity, host string) []string {
|
||||
return append(sshOpts(identity), host)
|
||||
}
|
||||
|
||||
func run(ctx context.Context, name string, args ...string) error {
|
||||
@@ -128,23 +139,24 @@ func agentBinary(explicit string) (string, func(), error) {
|
||||
return path, func() { os.RemoveAll(dir) }, nil
|
||||
}
|
||||
|
||||
func push(ctx context.Context, host, agentBin, resticBin string) error {
|
||||
if err := run(ctx, "ssh", append(sshBase(host), "mkdir -p "+remoteDir)...); err != nil {
|
||||
func push(ctx context.Context, identity, host, agentBin, resticBin string) error {
|
||||
if err := run(ctx, "ssh", append(sshBase(identity, host), "mkdir -p "+remoteDir)...); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := run(ctx, "scp", "-q", agentBin, host+":"+remoteDir+"/bench-agent"); err != nil {
|
||||
scp := append([]string{"-q"}, sshOpts(identity)...)
|
||||
if err := run(ctx, "scp", append(scp, agentBin, host+":"+remoteDir+"/bench-agent")...); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := run(ctx, "scp", "-q", resticBin, host+":"+remoteDir+"/restic"); err != nil {
|
||||
if err := run(ctx, "scp", append(scp, resticBin, host+":"+remoteDir+"/restic")...); err != nil {
|
||||
return err
|
||||
}
|
||||
return run(ctx, "ssh", append(sshBase(host), "chmod +x "+remoteDir+"/bench-agent "+remoteDir+"/restic")...)
|
||||
return run(ctx, "ssh", append(sshBase(identity, host), "chmod +x "+remoteDir+"/bench-agent "+remoteDir+"/restic")...)
|
||||
}
|
||||
|
||||
// drive runs the remote agent, feeding Config over stdin and consuming the
|
||||
// event stream from stdout. The agent's stderr passes straight through.
|
||||
func drive(ctx context.Context, host string, cfg Config) (*RunResult, error) {
|
||||
cmd := exec.CommandContext(ctx, "ssh", append(sshBase(host), remoteDir+"/bench-agent")...)
|
||||
func drive(ctx context.Context, identity, host string, cfg Config) (*RunResult, error) {
|
||||
cmd := exec.CommandContext(ctx, "ssh", append(sshBase(identity, host), remoteDir+"/bench-agent")...)
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
stdin, err := cmd.StdinPipe()
|
||||
|
||||
@@ -32,6 +32,7 @@ type benchFlags struct {
|
||||
|
||||
host string
|
||||
fromHere bool
|
||||
sshIdentity string
|
||||
agentBin string
|
||||
repo string
|
||||
repoID string
|
||||
@@ -57,6 +58,7 @@ func (f *benchFlags) registerCommon(c *cobra.Command) {
|
||||
f.admin.register(c)
|
||||
c.Flags().StringVar(&f.host, "host", "", "ssh destination of the management host (default: the region's first mgmt host from discovery)")
|
||||
c.Flags().BoolVar(&f.fromHere, "from-here", false, "run the benchmark on this machine (no ssh; agent runs in-process)")
|
||||
c.Flags().StringVar(&f.sshIdentity, "ssh-identity", "", "ssh private key for the management host (default: ssh agent/config)")
|
||||
c.Flags().StringVar(&f.agentBin, "agent-bin", "", "local linux/amd64 bench-agent binary (default: the embedded one)")
|
||||
c.Flags().StringVar(&f.repo, "repo", "", "restic repository URL; skips admin-api provisioning (default $RESTIC_REPOSITORY, else a repo is created via admin-api)")
|
||||
c.Flags().StringVar(&f.repoID, "repo-id", "", "existing repository id; a fresh URL is minted via admin-api")
|
||||
@@ -177,8 +179,17 @@ func (f *benchFlags) runBench(cmd *cobra.Command, defaultPhases []string) error
|
||||
}
|
||||
|
||||
if cfg.Repo == "" {
|
||||
if err := loadTopo(); err != nil {
|
||||
return err
|
||||
// Topology is only needed to derive the admin URL; an explicit
|
||||
// --admin-url / $YUCTL_ADMIN_API_URL skips state access entirely
|
||||
// (the context file alone names the token-cache partition).
|
||||
if f.admin.adminURL == "" && os.Getenv("YUCTL_ADMIN_API_URL") == "" {
|
||||
if err := loadTopo(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if cc == nil {
|
||||
if cc, err = requireContext(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
client, _, err := f.admin.adminLogin(ctx, cmd, cc, topo)
|
||||
if err != nil {
|
||||
@@ -219,7 +230,7 @@ func (f *benchFlags) runBench(cmd *cobra.Command, defaultPhases []string) error
|
||||
outPath = ""
|
||||
}
|
||||
|
||||
opts := bench.RunOpts{Host: host, AgentBin: f.agentBin, Config: cfg, Out: outPath}
|
||||
opts := bench.RunOpts{Host: host, SSHIdentity: f.sshIdentity, AgentBin: f.agentBin, Config: cfg, Out: outPath}
|
||||
if f.fromHere {
|
||||
_, err = bench.RunHere(ctx, opts)
|
||||
} else {
|
||||
|
||||
Generated
+6
@@ -102,6 +102,9 @@ catalogs:
|
||||
'@opentelemetry/propagator-jaeger':
|
||||
specifier: ^2.5.0
|
||||
version: 2.7.1
|
||||
'@opentelemetry/resources':
|
||||
specifier: ^2.5.0
|
||||
version: 2.7.1
|
||||
'@opentelemetry/sdk-node':
|
||||
specifier: ^0.217.0
|
||||
version: 0.217.0
|
||||
@@ -419,6 +422,9 @@ importers:
|
||||
'@opentelemetry/propagator-jaeger':
|
||||
specifier: 'catalog:'
|
||||
version: 2.7.1(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources':
|
||||
specifier: 'catalog:'
|
||||
version: 2.7.1(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-node':
|
||||
specifier: 'catalog:'
|
||||
version: 0.217.0(@opentelemetry/api@1.9.0)
|
||||
|
||||
@@ -43,6 +43,7 @@ catalog:
|
||||
'@opentelemetry/instrumentation-pino': ^0.57.0
|
||||
'@opentelemetry/propagator-b3': ^2.5.0
|
||||
'@opentelemetry/propagator-jaeger': ^2.5.0
|
||||
'@opentelemetry/resources': ^2.5.0
|
||||
'@opentelemetry/sdk-node': ^0.217.0
|
||||
'@playwright/test': 1.59.1
|
||||
'@sveltejs/adapter-auto': ^7.0.0
|
||||
|
||||
@@ -62,6 +62,10 @@ locals {
|
||||
hubble = cidrhost(module.addr_site.lb_internal_cidr, 16)
|
||||
# yucca-admin-api (namespace yucca), routed via the same netops gateway.
|
||||
admin = cidrhost(module.addr_site.lb_internal_cidr, 16)
|
||||
# michael (restic) internal gateway VIP — second IP on the gw envoy Service
|
||||
# (kubernetes/apps/prod/htz-fsn1/gw-proxy/), for on-net clients (mgmt hosts,
|
||||
# yuctl tools bench) since the public VIP aggregate isn't routed inside Hetzner.
|
||||
gw = cidrhost(module.addr_site.lb_internal_cidr, 17)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,9 @@ variable "groups" {
|
||||
}
|
||||
server_admins = {
|
||||
description = "Login access to provisioned servers (e.g. the ceph nodes)."
|
||||
server = { sudo = "ALL" }
|
||||
# NOPASSWD: these accounts authenticate by SSH key only and have no local
|
||||
# passwords to type; passworded sudo just blocks remote automation.
|
||||
server = { sudo = "NOPASSWD:ALL" }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user