From b7bbe2e80e4dc6ed2348d8005af3935f2fcc19b9 Mon Sep 17 00:00:00 2001 From: Antoine Lecompte <38678863+nutgood@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:53:59 -0400 Subject: [PATCH] chore(yuctl): restructure and make pretty (#485) --- .gitignore | 2 +- .mise/tasks/yuctl/build | 2 +- packages/yuctl/README.md | 152 +++--- .../{internal => }/adminapi/allowlist.go | 0 .../yuctl/{internal => }/adminapi/cache.go | 6 +- .../yuctl/{internal => }/adminapi/features.go | 0 .../yuctl/{internal => }/adminapi/http.go | 0 .../yuctl/{internal => }/adminapi/login.go | 0 .../yuctl/{internal => }/adminapi/repos.go | 0 .../yuctl/{internal => }/adminapi/settings.go | 0 .../yuctl/{internal => }/adminapi/users.go | 15 + .../{internal/ceph => cephhealth}/ceph.go | 8 +- .../yuctl/{internal/cli => cli/ceph}/ceph.go | 65 +-- .../{internal/cli => cli/config}/config.go | 88 ++- .../cli => cli/features}/features.go | 57 +- .../{internal/cli => cli/infra}/infra.go | 39 +- packages/yuctl/cli/login.go | 51 ++ packages/yuctl/{internal => }/cli/root.go | 64 ++- packages/yuctl/{internal => }/cli/select.go | 26 +- .../cli/tools.go => cli/tools/bench/bench.go} | 307 +++++------ .../tools/fleetbench/fleetbench.go} | 232 ++++---- packages/yuctl/cli/tools/fleetbench/view.go | 163 ++++++ packages/yuctl/cli/tools/tools.go | 19 + packages/yuctl/cli/tools/warp/view.go | 114 ++++ .../tools/warp/view_test.go} | 32 +- .../{internal/cli => cli/tools/warp}/warp.go | 143 +++-- .../yuctl/cli/users/allowlist/allowlist.go | 176 ++++++ .../cli/users/connections/connections.go | 57 ++ packages/yuctl/cli/users/features/features.go | 133 +++++ packages/yuctl/cli/users/users.go | 124 +++++ packages/yuctl/cmd/bench-agent/main.go | 30 +- packages/yuctl/cmdutil/admin.go | 147 +++++ packages/yuctl/cmdutil/factory.go | 50 ++ .../{internal/context => ctxstore}/context.go | 4 +- .../yuctl/{internal => }/discovery/cache.go | 4 +- .../{internal => }/discovery/cache_test.go | 2 +- .../{internal => }/discovery/discovery.go | 13 +- .../discovery/discovery_test.go | 2 +- packages/yuctl/{internal => }/do/do.go | 8 +- packages/yuctl/fleet/fleet.go | 82 +++ .../benchwide => fleet/fleetbench}/deploy.go | 129 ++--- .../fleetbench/fleetbench_test.go} | 14 +- packages/yuctl/fleet/fleetbench/render.go | 54 ++ .../benchwide => fleet/fleetbench}/run.go | 250 ++++----- .../benchwide => fleet/fleetbench}/session.go | 165 ++---- .../yuctl/{internal => fleet}/warp/cleanup.go | 5 +- .../yuctl/{internal => fleet}/warp/deploy.go | 0 .../warp/manifests/cleanup-job.yaml | 0 .../warp/manifests/deployment.yaml | 0 .../warp/manifests/namespace.yaml | 0 .../warp/manifests/secret.yaml | 0 .../yuctl/{internal => fleet}/warp/run.go | 180 +++---- .../yuctl/{internal => fleet}/warp/session.go | 4 +- .../{internal => fleet}/warp/warp_test.go | 0 packages/yuctl/internal/cli/adminauth.go | 169 ------ packages/yuctl/internal/cli/benchwideui.go | 227 -------- packages/yuctl/internal/cli/users.go | 506 ------------------ packages/yuctl/internal/cli/warpui.go | 194 ------- packages/yuctl/main.go | 4 +- .../yuctl/{internal => }/netdev/netdev.go | 2 +- .../{internal => }/netdev/netdev_test.go | 0 packages/yuctl/{internal => }/op/op.go | 0 packages/yuctl/{internal => }/op/op_test.go | 0 packages/yuctl/{internal => }/provider/do.go | 4 +- .../yuctl/{internal => }/provider/hetzner.go | 6 +- .../yuctl/{internal => }/provider/provider.go | 2 +- .../{internal/bench => resticbench}/agent.go | 2 +- .../bench => resticbench}/bench_test.go | 2 +- .../{internal/bench => resticbench}/config.go | 2 +- .../bench => resticbench}/dataset.go | 2 +- .../bench => resticbench}/embedded_off.go | 2 +- .../bench => resticbench}/embedded_on.go | 2 +- .../{internal/bench => resticbench}/events.go | 22 +- .../{internal/bench => resticbench}/fetch.go | 2 +- .../bench => resticbench}/loadgen.go | 10 +- .../bench => resticbench}/loadgen_test.go | 2 +- .../bench => resticbench}/orchestrator.go | 95 +--- .../{internal/bench => resticbench}/restic.go | 2 +- .../bench => resticbench}/results.go | 2 +- .../{internal/bench => resticbench}/sizes.go | 2 +- packages/yuctl/sshx/sshx.go | 158 ++++++ .../yuctl/{internal => }/state/contract.go | 0 .../{internal => }/state/contract_test.go | 0 .../yuctl/{internal/k8s => talos}/talos.go | 10 +- packages/yuctl/ui/iostreams.go | 20 + packages/yuctl/ui/theme.go | 26 + packages/yuctl/ui/widgets.go | 61 +++ packages/yuctl/ui/widgets_test.go | 18 + 88 files changed, 2458 insertions(+), 2316 deletions(-) rename packages/yuctl/{internal => }/adminapi/allowlist.go (100%) rename packages/yuctl/{internal => }/adminapi/cache.go (94%) rename packages/yuctl/{internal => }/adminapi/features.go (100%) rename packages/yuctl/{internal => }/adminapi/http.go (100%) rename packages/yuctl/{internal => }/adminapi/login.go (100%) rename packages/yuctl/{internal => }/adminapi/repos.go (100%) rename packages/yuctl/{internal => }/adminapi/settings.go (100%) rename packages/yuctl/{internal => }/adminapi/users.go (89%) rename packages/yuctl/{internal/ceph => cephhealth}/ceph.go (94%) rename packages/yuctl/{internal/cli => cli/ceph}/ceph.go (57%) rename packages/yuctl/{internal/cli => cli/config}/config.go (73%) rename packages/yuctl/{internal/cli => cli/features}/features.go (60%) rename packages/yuctl/{internal/cli => cli/infra}/infra.go (66%) create mode 100644 packages/yuctl/cli/login.go rename packages/yuctl/{internal => }/cli/root.go (66%) rename packages/yuctl/{internal => }/cli/select.go (63%) rename packages/yuctl/{internal/cli/tools.go => cli/tools/bench/bench.go} (52%) rename packages/yuctl/{internal/cli/benchwide.go => cli/tools/fleetbench/fleetbench.go} (55%) create mode 100644 packages/yuctl/cli/tools/fleetbench/view.go create mode 100644 packages/yuctl/cli/tools/tools.go create mode 100644 packages/yuctl/cli/tools/warp/view.go rename packages/yuctl/{internal/cli/warpui_test.go => cli/tools/warp/view_test.go} (57%) rename packages/yuctl/{internal/cli => cli/tools/warp}/warp.go (73%) create mode 100644 packages/yuctl/cli/users/allowlist/allowlist.go create mode 100644 packages/yuctl/cli/users/connections/connections.go create mode 100644 packages/yuctl/cli/users/features/features.go create mode 100644 packages/yuctl/cli/users/users.go create mode 100644 packages/yuctl/cmdutil/admin.go create mode 100644 packages/yuctl/cmdutil/factory.go rename packages/yuctl/{internal/context => ctxstore}/context.go (96%) rename packages/yuctl/{internal => }/discovery/cache.go (98%) rename packages/yuctl/{internal => }/discovery/cache_test.go (98%) rename packages/yuctl/{internal => }/discovery/discovery.go (97%) rename packages/yuctl/{internal => }/discovery/discovery_test.go (99%) rename packages/yuctl/{internal => }/do/do.go (97%) create mode 100644 packages/yuctl/fleet/fleet.go rename packages/yuctl/{internal/benchwide => fleet/fleetbench}/deploy.go (64%) rename packages/yuctl/{internal/benchwide/benchwide_test.go => fleet/fleetbench/fleetbench_test.go} (93%) create mode 100644 packages/yuctl/fleet/fleetbench/render.go rename packages/yuctl/{internal/benchwide => fleet/fleetbench}/run.go (61%) rename packages/yuctl/{internal/benchwide => fleet/fleetbench}/session.go (53%) rename packages/yuctl/{internal => fleet}/warp/cleanup.go (96%) rename packages/yuctl/{internal => fleet}/warp/deploy.go (100%) rename packages/yuctl/{internal => fleet}/warp/manifests/cleanup-job.yaml (100%) rename packages/yuctl/{internal => fleet}/warp/manifests/deployment.yaml (100%) rename packages/yuctl/{internal => fleet}/warp/manifests/namespace.yaml (100%) rename packages/yuctl/{internal => fleet}/warp/manifests/secret.yaml (100%) rename packages/yuctl/{internal => fleet}/warp/run.go (75%) rename packages/yuctl/{internal => fleet}/warp/session.go (99%) rename packages/yuctl/{internal => fleet}/warp/warp_test.go (100%) delete mode 100644 packages/yuctl/internal/cli/adminauth.go delete mode 100644 packages/yuctl/internal/cli/benchwideui.go delete mode 100644 packages/yuctl/internal/cli/users.go delete mode 100644 packages/yuctl/internal/cli/warpui.go rename packages/yuctl/{internal => }/netdev/netdev.go (94%) rename packages/yuctl/{internal => }/netdev/netdev_test.go (100%) rename packages/yuctl/{internal => }/op/op.go (100%) rename packages/yuctl/{internal => }/op/op_test.go (100%) rename packages/yuctl/{internal => }/provider/do.go (97%) rename packages/yuctl/{internal => }/provider/hetzner.go (98%) rename packages/yuctl/{internal => }/provider/provider.go (98%) rename packages/yuctl/{internal/bench => resticbench}/agent.go (99%) rename packages/yuctl/{internal/bench => resticbench}/bench_test.go (99%) rename packages/yuctl/{internal/bench => resticbench}/config.go (98%) rename packages/yuctl/{internal/bench => resticbench}/dataset.go (99%) rename packages/yuctl/{internal/bench => resticbench}/embedded_off.go (91%) rename packages/yuctl/{internal/bench => resticbench}/embedded_on.go (95%) rename packages/yuctl/{internal/bench => resticbench}/events.go (61%) rename packages/yuctl/{internal/bench => resticbench}/fetch.go (99%) rename packages/yuctl/{internal/bench => resticbench}/loadgen.go (97%) rename packages/yuctl/{internal/bench => resticbench}/loadgen_test.go (98%) rename packages/yuctl/{internal/bench => resticbench}/orchestrator.go (67%) rename packages/yuctl/{internal/bench => resticbench}/restic.go (99%) rename packages/yuctl/{internal/bench => resticbench}/results.go (99%) rename packages/yuctl/{internal/bench => resticbench}/sizes.go (99%) create mode 100644 packages/yuctl/sshx/sshx.go rename packages/yuctl/{internal => }/state/contract.go (100%) rename packages/yuctl/{internal => }/state/contract_test.go (100%) rename packages/yuctl/{internal/k8s => talos}/talos.go (91%) create mode 100644 packages/yuctl/ui/iostreams.go create mode 100644 packages/yuctl/ui/theme.go create mode 100644 packages/yuctl/ui/widgets.go create mode 100644 packages/yuctl/ui/widgets_test.go diff --git a/.gitignore b/.gitignore index b4f38cf2..7f6c10c0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ node_modules mise.local.toml dist/ -packages/yuctl/internal/bench/bench-agent-linux-amd64.gz +packages/yuctl/resticbench/bench-agent-linux-amd64.gz packages/michael/michael # k3d/Tilt dev stack — Helm builds subchart snapshots on the fly; the .dev/ diff --git a/.mise/tasks/yuctl/build b/.mise/tasks/yuctl/build index e30ff155..2dc3e92c 100755 --- a/.mise/tasks/yuctl/build +++ b/.mise/tasks/yuctl/build @@ -9,6 +9,6 @@ cd packages/yuctl # `yuctl tools bench` is self-contained. Plain `go build` still works without # the tag — the embedded agent is then absent and --agent-bin is required. CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../dist/bench-agent-linux-amd64 ./cmd/bench-agent -gzip -9 -n -c ../../dist/bench-agent-linux-amd64 > internal/bench/bench-agent-linux-amd64.gz +gzip -9 -n -c ../../dist/bench-agent-linux-amd64 > resticbench/bench-agent-linux-amd64.gz go build -tags embedagent -o ../../dist/yuctl . diff --git a/packages/yuctl/README.md b/packages/yuctl/README.md index a56a8f01..ef5c6732 100644 --- a/packages/yuctl/README.md +++ b/packages/yuctl/README.md @@ -12,12 +12,19 @@ references, never values). ## Conventions -Built to match `packages/michael`: `module yuctl`, Go 1.25, `main.go` + -`internal/`, `aws-sdk-go-v2` for S3, `rs/zerolog` for logging. The one -documented divergence is **`spf13/cobra`** for the nested subcommand tree -(michael is a single-purpose HTTP server and stays stdlib-only; yuctl is a -multi-verb CLI). There is no Dockerfile — yuctl is an operator CLI, not a -deployed service. +`module yuctl`, Go 1.25, `aws-sdk-go-v2` for S3, `rs/zerolog` for logging — +matching `packages/michael` — with two documented divergences: **`spf13/cobra`** +for the nested subcommand tree (michael is a single-purpose HTTP server and +stays stdlib-only; yuctl is a multi-verb CLI), and a **flat package layout** +with no `internal/` (yuctl is an unpublishable standalone module nothing else +imports, so the boundary bought nothing but a path segment). There is no +Dockerfile — yuctl is an operator CLI, not a deployed service. + +The command layer follows the gh/kubectl shape: `cli/` mirrors the command +tree one package per topic (`yuctl ceph …` → `cli/ceph`, `yuctl tools warp …` +→ `cli/tools/warp`), and every command receives a `cmdutil.Factory` carrying +the lazily-resolved shared dependencies (IO streams, selected context, +memoized topology, admin-api login) instead of re-deriving them per command. ## Build @@ -32,16 +39,29 @@ Or directly: `cd packages/yuctl && go build -o ../../dist/yuctl .` ``` packages/yuctl/ - main.go # entrypoint → cli.NewRootCmd().ExecuteContext - internal/ - cli/ # cobra command tree (root, select, ceph, infra, users) - discovery/ # S3 state reader + stack enumeration + topology queries - state/ # discovery output contract structs + tfstate parsing - op/ # `op read` / ReadToTempFile (0600) wrapper - context/ # ~/.config/yuctl/context.json {partition,region,ceph_cluster} - k8s/ # talosctl upgrade wrapper - ceph/ # RGW/dashboard health probe - adminapi/ # CLI loopback login + Bearer admin-api client + main.go # yuctl entrypoint → cli.NewRootCmd().ExecuteContext + cmd/bench-agent/ # second binary: the remote bench agent + cli/ # cobra wiring, one package per topic, mirrors the command tree + root.go select.go login.go + ceph/ infra/ config/ features/ + users/{allowlist,features,connections}/ + tools/{bench,fleetbench,warp}/ + cmdutil/ # Factory (IO, context, topology, admin login) + Confirm/OpenBrowser + ui/ # IOStreams, lipgloss theme, meter/sparkline widgets + fleet/ # shared fleet-tool engine: watch loop, history, parallel fan-out + warp/ # K8s-pod transport: warp runner fleet vs RGW + fleetbench/ # cloud-VM transport: restic client fleet vs michael + sshx/ # the one ssh/scp layer (multiplexing, retry, stdin secrets) + resticbench/ # bench agent + orchestrator + loadgen + restic runner + discovery/ # S3 state reader + stack enumeration + topology queries + state/ # discovery output contract structs + tfstate parsing + op/ # `op read` / ReadToTempFile (0600) wrapper + ctxstore/ # ~/.config/yuctl/context.json {partition,region,ceph_cluster} + talos/ # talosctl upgrade wrapper + cephhealth/ # RGW/dashboard health probe + adminapi/ # CLI loopback login + Bearer admin-api client + provider/ # cloud-VM providers (DO, Hetzner) for fleet-bench + do/ netdev/ # DigitalOcean plumbing; /proc/net/dev parsing ``` ## Command tree @@ -69,14 +89,14 @@ yuctl ├── bench restic e2e benchmark against michael, run from a mgmt host │ ├── compare render before/after deltas from two results files │ └── cleanup forget+prune every bench snapshot (timed) - ├── bench-do restic client fleet on DigitalOcean droplets vs michael - │ ├── deploy create/converge the droplet fleet (project yucca-bench) + ├── fleet-bench restic client fleet on cloud VMs (--provider do|hetzner) vs michael + │ ├── deploy create/converge the host fleet (project yucca-bench) │ ├── start launch the per-client backup loops (graceful restart) │ ├── status one-shot dashboard (throughput, transfer budget, clients) │ ├── watch live dashboard, continuously sampled │ ├── stop kill the load, collect + save the results JSON - │ ├── cleanup forget+prune every bench-do repo (from the droplets) - │ └── undeploy destroy the droplets and the ephemeral ssh key + │ ├── cleanup forget+prune every fleet-bench repo (from the hosts) + │ └── undeploy destroy the hosts and the ephemeral ssh key └── warp S3 load test fleet against the region's RGW gateways ├── deploy create/converge hostNetwork runner pods on the workers ├── start launch the load (graceful restart; non-stop by default) @@ -100,7 +120,7 @@ Global flags: `--log-level` (trace|debug|info|warn|error), `--log-format` - **bucket fallback**: `ListObjectsV2` on `yucca-tf-state` under prefix `yucca/`, keeping `*/terraform.tfstate` keys. 2. **Resolve live values** — `GetObject` each `terraform.tfstate` and parse - `.outputs.discovery.value` into `internal/state.Discovery`. Stacks with no + `.outputs.discovery.value` into `state.Discovery`. Stacks with no `discovery` output (pre-contract) or no applied state are skipped, not fatal. 3. **Query** the merged `Topology` (`HasRegion`, `Kubernetes`, `CephClusters`, `PrimaryRegion`, `RegionMeta`). @@ -240,58 +260,62 @@ The ssh session stays open for the whole run (keepalives set); run multi-hour benchmarks inside tmux. Pair the client numbers with the michael dashboard (TTFB, connection churn, S3 client metrics) for the server-side view. -## `tools bench-do` — DigitalOcean restic client fleet +## `tools fleet-bench` — cloud-VM restic client fleet -Drives michael from the outside: N DigitalOcean droplets each running real -restic clients over the public internet — the actual external-user path (DNS, -edge, michael, RGW), unlike `bench` (mgmt host on the fabric) and `warp` -(in-cluster, straight at RGW). Fleet lifecycle mirrors `warp`. +Drives michael from the outside: N cloud VMs (`--provider do|hetzner`) each +running real restic clients over the public internet — the actual +external-user path (DNS, edge, michael, RGW), unlike `bench` (mgmt host on +the fabric) and `warp` (in-cluster, straight at RGW). Fleet lifecycle mirrors +`warp`; fleets are per provider × partition, so several providers can load +michael at once. ```bash yuctl select prod@htz-fsn1 yuctl login -yuctl tools bench-do start --droplets 6 --clients-per-droplet 2 \ +yuctl tools fleet-bench start --hosts 6 --clients-per-host 2 \ --obj-size 64MiB --duration 2h --label big-packs # auto-deploys (confirms cost first) -yuctl tools bench-do watch # live dashboard: Gbps, transfer budget bars, client loops -yuctl tools bench-do stop # kill the load, save bench-do-