From 7e86af28b3f04c48e6fe2128384098b640f2893d Mon Sep 17 00:00:00 2001
From: Ryan Hughes <ryan@heyoodle.com>
Date: Fri, 9 Oct 2026 09:41:43 -0400
Subject: [PATCH] Attach operator keys to builders so none gets an emailed root
 password

The controller created every builder without a DigitalOcean account key, so
DigitalOcean set a root password, emailed it to the token's owner, and turned
password logins on: one email and one password-reachable box per builder,
146 of them on Oct 8.

DO_SSH_KEYS lists account key IDs attached at creation (Ryan, DHH and Emir
in the template). It needs ssh_key:read on the controller's token; without it
every create is refused with 403, so it stays empty until the token has it.
Builders also turn password SSH off in cloud-init, which holds even while no
account key is attached, and Emir joins the operators whose GitHub keys reach
root.
---
 ci/README.md                             |  4 +++-
 ci/controller-box/controller.env.example |  5 +++++
 ci/controller-box/create.sh              |  2 +-
 ci/controller.sh                         | 10 ++++++++--
 ci/runner-cloud-init.yaml                |  3 +++
 tests/controller.sh                      |  6 ++++++
 6 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/ci/README.md b/ci/README.md
index c38d944c..4ca988cc 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -75,6 +75,7 @@ Edit `/etc/omarchy-controller.env` on the box. The next tick reads it.
 | `REGIONS` | empty | Regions to try first; empty means any |
 | `MAX_AGE_MINUTES` | 200 | Delete a droplet older than this |
 | `MAX_BOOT_MINUTES` | 10 | Delete a droplet still provisioning after this |
+| `DO_SSH_KEYS` | Ryan, DHH and Emir's account key IDs | Attached to every builder. Without one, DigitalOcean emails a root password per builder |
 
 ### Change the controller
 
@@ -110,7 +111,8 @@ DIGITALOCEAN_TOKEN=<account that pays for builders> GITHUB_TOKEN=<fine-grained P
 - The GitHub PAT is fine-grained and scoped to this repository: Actions read,
   Administration read and write (for runner registration tokens).
 - The DigitalOcean token is written into the box's env file, so it is the
-  account that pays.
+  account that pays. It needs ssh_key:read to attach `DO_SSH_KEYS`; without
+  it every create is refused with 403.
 - `ADMIN_GITHUB_USERS` names whose GitHub SSH keys get root on the box and
   the builders. Set it; the default is a fixed list in `create.sh`.
 - The script refuses to create a second controller.
diff --git a/ci/controller-box/controller.env.example b/ci/controller-box/controller.env.example
index c2b59795..31f2a2bb 100644
--- a/ci/controller-box/controller.env.example
+++ b/ci/controller-box/controller.env.example
@@ -17,3 +17,8 @@ LOCK=/run/omarchy-controller/lock
 # Operator public keys for root on every builder droplet (JSON array).
 # create.sh fills this from the operators' GitHub keys.
 SSH_KEYS_JSON=[]
+# DigitalOcean account SSH key IDs attached to every builder (JSON array), so
+# DigitalOcean does not email a root password for each one. Needs ssh_key:read
+# on DIGITALOCEAN_TOKEN. List them with: doctl compute ssh-key list
+# Ryan R. Hughes, DHH, Emir.
+DO_SSH_KEYS=[59143572,59143588,59779105]
diff --git a/ci/controller-box/create.sh b/ci/controller-box/create.sh
index 9ec4c3dc..9eb2d68f 100755
--- a/ci/controller-box/create.sh
+++ b/ci/controller-box/create.sh
@@ -18,7 +18,7 @@ SSH_KEYS=${SSH_KEYS:-[]}
 # Public keys authorized for root: the operators' GitHub keys, fetched at
 # creation so the box never depends on an ssh_key API scope. Override with
 # ADMIN_GITHUB_USERS.
-ADMIN_GITHUB_USERS=${ADMIN_GITHUB_USERS:-ryanrhughes dhh}
+ADMIN_GITHUB_USERS=${ADMIN_GITHUB_USERS:-ryanrhughes dhh emirb}
 ssh_keys_json=$(for u in $ADMIN_GITHUB_USERS; do curl -fsS "https://github.com/$u.keys"; done | jq -R . | jq -sc .)
 [[ $(jq length <<<"$ssh_keys_json") -gt 0 ]] || { echo "no ssh keys fetched for $ADMIN_GITHUB_USERS" >&2; exit 1; }
 
diff --git a/ci/controller.sh b/ci/controller.sh
index d69f4e3a..1e5383f1 100755
--- a/ci/controller.sh
+++ b/ci/controller.sh
@@ -40,6 +40,12 @@ CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml}
 # Operator public keys authorized on every builder (JSON array of strings).
 # The box's env file carries them; empty means no root login.
 SSH_KEYS_JSON=${SSH_KEYS_JSON:-[]}
+# DigitalOcean account SSH key IDs or fingerprints attached at creation (JSON
+# array). A droplet created with none gets a root password that DigitalOcean
+# emails to the token's owner, once per builder. Attaching one needs the
+# ssh_key:read scope on DIGITALOCEAN_TOKEN; without it every create is
+# refused with 403.
+DO_SSH_KEYS=${DO_SSH_KEYS:-[]}
 LOCK=${LOCK:-/tmp/omarchy-controller.lock}
 
 log() { echo "$(date '+%F %T') $*"; }
@@ -152,8 +158,8 @@ create_droplet() {
   while read -r size region; do
     [[ -n $size ]] || continue
     body=$(jq -n --arg name "$name" --arg region "$region" --arg size "$size" --arg image "$IMAGE" \
-      --arg tag "$TAG" --arg ud "$userdata" \
-      '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
+      --arg tag "$TAG" --arg ud "$userdata" --argjson keys "$DO_SSH_KEYS" \
+      '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, ssh_keys:$keys, monitoring:false}')
     log "creating $name ($size in $region)"
     if response=$(do_api droplets -X POST -d "$body"); then
       jq -r '"created droplet \(.droplet.id)"' <<< "$response"
diff --git a/ci/runner-cloud-init.yaml b/ci/runner-cloud-init.yaml
index 1cb409e7..17da958b 100644
--- a/ci/runner-cloud-init.yaml
+++ b/ci/runner-cloud-init.yaml
@@ -14,7 +14,10 @@
 
 # Operators can reach a builder by key while it lives; it powers off after
 # one job anyway. Keys are substituted by the controller (__SSH_KEYS_JSON__).
+# Never by password: DigitalOcean sets and emails one when no account key is
+# attached, and its vendor data turns password logins on to match.
 disable_root: false
+ssh_pwauth: false
 chpasswd:
   expire: false
 ssh_authorized_keys: __SSH_KEYS_JSON__
diff --git a/tests/controller.sh b/tests/controller.sh
index cd21aa6a..a7503456 100755
--- a/tests/controller.sh
+++ b/tests/controller.sh
@@ -105,6 +105,12 @@ jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.use
   && echo "PASS: create body carries tag, size, substituted user-data" \
   || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; }
 
+# Account keys reach the create, so DigitalOcean emails no root password.
+CANDIDATES=""; DO_SSH_KEYS='[123]' create_droplet >/dev/null
+jq -e '.ssh_keys == [123]' "$BODY_FILE" >/dev/null \
+  && echo "PASS: create body attaches DO_SSH_KEYS" \
+  || { echo "FAIL: create body ssh_keys"; jq .ssh_keys "$BODY_FILE"; exit 1; }
+
 # Sizes are tried in SIZES order, each in the regions DigitalOcean lists it
 # in stock (REGIONS first); a 422 falls through to the next pair with the
 # refusal's message logged, and a refused pair is not retried in the tick.