initial commit

scripts/darwin-ci/README.md

@@ -0,0 +1,61 @@
+# darwin CI agents
+
+Provisioning for the macOS test agents on queue `test-darwin`.
+
+Two modes:
+
+- `tart`: for Apple Silicon hosts with memory to spare. The host runs only
+  `buildkite-agent` and [Tart](https://tart.run); every test job runs in a
+  fresh macOS guest cloned from a baked image and deleted afterwards.
+- `bare`: for Intel hosts (Tart cannot virtualize macOS on Intel) and small
+  Apple Silicon hosts. The bun toolchain and `scripts/agent.mjs` service run
+  on the host itself.
+
+Agents tag themselves `os=darwin arch=... release=<macOS major> release-tier=...`
+and `.buildkite/ci.mjs` selects on those. In tart mode `release` is the
+guest's macOS version, and a guest cannot be newer than its host.
+
+## Layout
+
+```
+host.sh            first contact: installs brew and bun, then runs `main.ts provision`
+main.ts            provision | setup-user | bake | install-agent
+lib/               host hardening, tailscale, the unprivileged CI user, tart, bake, agent config
+hooks/             agent hooks for tart hosts (command, pre-exit, environment)
+guest/bake.sh      runs inside the guest once, at bake time
+guest/job.sh       runs inside the guest for every job
+```
+
+`provision <hostname> tart` disables remote management, makes sshd key-only,
+joins the tailnet, installs `buildkite-agent` and Tart, creates an
+unprivileged auto-login user (Virtualization.framework needs a console
+session), bakes the guest image from a public base image plus
+`scripts/bootstrap.sh`, and starts the agent as that user with `hooks/` as
+its hooks path. It asks for one reboot the first time and is re-run after it.
+
+`provision <hostname> bare` does the same host setup, then runs
+`scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service.
+
+`bake` is safe on a live host: it builds a staging image and swaps it in only
+after the toolchain verifies. Re-run it when toolchain pins move.
+
+## Bringing up a host
+
+Prerequisites on a freshly imaged host: an admin account you can ssh into
+with a key, passwordless sudo for it, the host's address on the agent
+token's IP allowlist, and the agent token written to the root-only file named
+in `lib/config.ts` (or `DARWIN_CI_TOKEN_FILE`).
+
+```sh
+scp -r scripts/darwin-ci <admin>@<host>:
+ssh <admin>@<host> 'darwin-ci/host.sh <hostname> tart --tags <tailscale tags>'
+```
+
+Approve the Tailscale login it prints, reboot when it asks, run the same
+command again, and check the agent appears under `queue=test-darwin`.
+
+## Removing a host
+
+Unload the agent (`launchctl bootout` the `com.buildkite.buildkite-agent`
+job), drop the host from the token allowlist and the tailnet, and release it.
+Nothing on a host needs preserving.

scripts/darwin-ci/guest/bake.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Runs inside a fresh guest to turn the base image into bun-ci-base. Args: <bun repo> <ref> <buildkite-agent version> <tools to verify>
+set -euo pipefail
+exec </dev/null
+repo=$1 ref=$2 agent_version=$3 toolchain=$4
+
+printf 'PasswordAuthentication no\nKbdInteractiveAuthentication no\n' | sudo tee /etc/ssh/sshd_config.d/000-hardening.conf >/dev/null
+sudo /System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart -deactivate -stop >/dev/null 2>&1 || true
+sudo launchctl disable system/com.apple.screensharing 2>/dev/null || true
+
+eval "$(/opt/homebrew/bin/brew shellenv)"
+# the base image ships a brew node that shadows the version bootstrap.sh pins
+brew uninstall --force --ignore-dependencies node node@24 node@22 >/dev/null 2>&1 || true
+rm -f /opt/homebrew/bin/node /opt/homebrew/bin/npm /opt/homebrew/bin/npx
+touch ~/.profile ~/.zshrc ~/.bash_profile
+
+rm -rf ~/bun-bootstrap
+git clone -q --depth=1 --branch "$ref" "$repo" ~/bun-bootstrap
+(cd ~/bun-bootstrap && ./scripts/bootstrap.sh) || echo "bootstrap.sh exited $?; verifying toolchain"
+
+missing=0
+for tool in $toolchain; do
+  path=$(bash -lc "command -v $tool" || true)
+  printf '  %-10s %s\n' "$tool" "${path:-MISSING}"
+  [ -n "$path" ] || missing=1
+done
+[ "$missing" = 0 ] || exit 1
+
+tmp=$(mktemp -d)
+curl -fsSL -o "$tmp/agent.tgz" "https://github.com/buildkite/agent/releases/download/v$agent_version/buildkite-agent-darwin-arm64-$agent_version.tar.gz"
+tar -xzf "$tmp/agent.tgz" -C "$tmp"
+sudo mkdir -p /usr/local/bin
+sudo install -m 755 "$tmp/buildkite-agent" /usr/local/bin/buildkite-agent
+
+mkdir -p ~/work
+rm -rf "$tmp" ~/bun-bootstrap ~/Library/Caches/Homebrew/downloads
+echo BAKE_OK

scripts/darwin-ci/guest/job.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Runs inside the per-job guest. Expects ~/job.env (exported by the command hook) and the checkout at ~/work.
+set -u
+export PATH=/usr/local/bin:/opt/homebrew/bin:/opt/rust/bin:$PATH
+# bootstrap.sh installs rustup into /opt/rust and only exports these from the login profile, which this script
+# does not source; without them the proxies in /opt/rust/bin look for a toolchain in ~/.rustup and find none.
+export RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust
+export BUILDKITE_BUILD_CHECKOUT_PATH=$HOME/work
+set -a; source ~/job.env; set +a
+cd ~/work
+
+echo "[guest] macOS $(sw_vers -productVersion) shard=${BUILDKITE_PARALLEL_JOB:-0}/${BUILDKITE_PARALLEL_JOB_COUNT:-1}"
+bun install >/dev/null 2>&1 || true
+(cd test && bun install >/dev/null 2>&1) || true
+
+shard=()
+[ -n "${BUILDKITE_PARALLEL_JOB:-}" ] && shard=(--shard="$BUILDKITE_PARALLEL_JOB" --max-shards="$BUILDKITE_PARALLEL_JOB_COUNT")
+
+# the runner can leak servers that keep node alive after the run, so wait on it directly and stream its log
+log=$HOME/runner.out
+: >"$log"
+$BUILDKITE_COMMAND "${shard[@]}" >"$log" 2>&1 &
+runner=$!
+tail -f "$log" &
+tailer=$!
+wait $runner
+status=$?
+kill $tailer 2>/dev/null
+exit $status

scripts/darwin-ci/hooks/command

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /usr/local/bin/bun "$BUILDKITE_HOOKS_PATH/command.ts"

scripts/darwin-ci/hooks/command.ts

@@ -0,0 +1,95 @@
+import { $ } from "bun";
+import { unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { config } from "../lib/config";
+import { guest } from "../lib/guest";
+import { fail } from "../lib/shell";
+import { tart } from "../lib/tart";
+
+const env = process.env;
+const command = env.BUILDKITE_COMMAND ?? "";
+const jobId = env.BUILDKITE_JOB_ID ?? String(process.pid);
+const checkout = env.BUILDKITE_BUILD_CHECKOUT_PATH ?? fail("no BUILDKITE_BUILD_CHECKOUT_PATH");
+const vm = `bk-${jobId}`;
+
+const hostOnlyEnv = new Set([
+  "BUILDKITE_AGENT_JOB_API_SOCKET",
+  "BUILDKITE_AGENT_JOB_API_TOKEN",
+  "BUILDKITE_BIN_PATH",
+  "BUILDKITE_BUILD_CHECKOUT_PATH",
+  "BUILDKITE_BUILD_PATH",
+  "BUILDKITE_ENV_FILE",
+  "BUILDKITE_HOOKS_PATH",
+  "BUILDKITE_PLUGINS_PATH",
+  "BUILDKITE_SOCKETS_PATH",
+]);
+
+if (!command.includes("runner.node.mjs")) {
+  console.log("--- running on host (not a test step)");
+  process.exit((await $`bash -c ${command}`.nothrow()).exitCode);
+}
+
+let cleaned = false;
+async function cleanup(): Promise<void> {
+  if (cleaned) return;
+  cleaned = true;
+  await tart.destroy(vm);
+}
+for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
+  process.on(signal, () => cleanup().finally(() => process.exit(130)));
+}
+
+let status = 1;
+try {
+  status = await runInGuest();
+} finally {
+  await cleanup();
+}
+process.exit(status);
+
+async function runInGuest(): Promise<number> {
+  console.log(`--- :tart: ${vm} from ${config.tart.image}`);
+  const ip = (await boot()) ?? (await retryBoot()) ?? fail(`guest never came up; see /tmp/${vm}.log`);
+  console.log(`    guest ${ip}`);
+  const g = guest(ip);
+
+  console.log("--- :package: sync checkout");
+  await g.syncTo(checkout, "work");
+  await g.push(join(import.meta.dir, "..", "guest", "job.sh"), "job.sh");
+  await pushEnv(g);
+
+  console.log(`+++ :test_tube: ${command}`);
+  const socket = "/tmp/bk-job-api.sock";
+  const forward = ["-o", "StreamLocalBindUnlink=yes", "-R", `${socket}:${env.BUILDKITE_AGENT_JOB_API_SOCKET}`];
+  const status = await g.run(
+    `BUILDKITE_AGENT_JOB_API_SOCKET=${socket} BUILDKITE_AGENT_JOB_API_TOKEN=${env.BUILDKITE_AGENT_JOB_API_TOKEN ?? ""} /bin/bash ~/job.sh`,
+    forward,
+  );
+
+  console.log("--- :outbox_tray: collect reports");
+  await g.collectReports("work", checkout);
+  return status;
+}
+
+async function boot(): Promise<string | undefined> {
+  await tart.cloneLocked(config.tart.image, vm);
+  tart.start(vm, `/tmp/${vm}.log`);
+  return tart.waitForSsh(vm);
+}
+
+// the first boot after a host restart can lose a race with vmnet coming up
+async function retryBoot(): Promise<string | undefined> {
+  console.log("    first boot failed; retrying once");
+  await tart.destroy(vm);
+  return boot();
+}
+
+async function pushEnv(g: ReturnType<typeof guest>): Promise<void> {
+  const lines = Object.entries(env)
+    .filter(([key]) => /^(BUILDKITE|BUN_|EXPECTED_PLATFORM_)|^(CI|ASAN_OPTIONS)$/.test(key) && !hostOnlyEnv.has(key))
+    .map(([key, value]) => `${key}=${$.escape(String(value ?? ""))}`);
+  const file = `/tmp/${vm}.env`;
+  writeFileSync(file, lines.join("\n") + "\n");
+  await g.push(file, "job.env");
+  unlinkSync(file);
+}

scripts/darwin-ci/hooks/environment

@@ -0,0 +1,2 @@
+#!/bin/bash
+export PATH="/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"

scripts/darwin-ci/hooks/pre-exit

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /usr/local/bin/bun "$BUILDKITE_HOOKS_PATH/pre-exit.ts"

scripts/darwin-ci/hooks/pre-exit.ts

@@ -0,0 +1,19 @@
+import { $ } from "bun";
+import { readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { succeeds } from "../lib/shell";
+import { tart } from "../lib/tart";
+
+await tart.destroy(`bk-${process.env.BUILDKITE_JOB_ID ?? "none"}`);
+
+// a bk-* guest with no `tart run` behind it belongs to a job that died without its cleanup running
+const vms = join(homedir(), ".tart", "vms");
+const tenMinutesAgo = Date.now() - 10 * 60_000;
+for (const name of readdirSync(vms).filter((name: string) => name.startsWith("bk-"))) {
+  const stat = statSync(join(vms, name), { throwIfNoEntry: false });
+  if (!stat || stat.mtimeMs > tenMinutesAgo) continue;
+  if (await succeeds($`pgrep -f ${`tart run ${name}`}`)) continue;
+  await tart.remove(name);
+  console.log(`pre-exit: reaped orphan guest ${name}`);
+}

scripts/darwin-ci/host.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# First contact with a freshly imaged host: installs brew and bun, then hands off to main.ts provision.
+set -euo pipefail
+here=$(cd "$(dirname "$0")" && pwd)
+sudo -n true 2>/dev/null || { echo "needs passwordless sudo for $(whoami)"; exit 1; }
+
+if [ "$(uname -m)" = arm64 ]; then prefix=/opt/homebrew; target=aarch64; else prefix=/usr/local; target=x64; fi
+
+if [ ! -x "$prefix/bin/brew" ]; then
+  NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+fi
+eval "$("$prefix/bin/brew" shellenv)"
+
+if [ ! -x /usr/local/bin/bun ]; then
+  tmp=$(mktemp -d)
+  curl -fsSL -o "$tmp/bun.zip" "https://github.com/oven-sh/bun/releases/latest/download/bun-darwin-$target.zip"
+  unzip -q "$tmp/bun.zip" -d "$tmp"
+  sudo mkdir -p /usr/local/bin
+  sudo install -m 755 "$tmp/bun-darwin-$target/bun" /usr/local/bin/bun
+  rm -rf "$tmp"
+fi
+
+exec /usr/local/bin/bun "$here/main.ts" provision "$@"

scripts/darwin-ci/lib/agent.ts

@@ -0,0 +1,116 @@
+import { $ } from "bun";
+import { join } from "node:path";
+import { ciUserHome, ciUserId } from "./ci-user";
+import { config, releaseTier } from "./config";
+import { plist } from "./launchd";
+import { consoleUser, fail, output, poll, sleep, succeeds, sudoRead, sudoWrite } from "./shell";
+
+const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin";
+const agentLabel = "com.buildkite.buildkite-agent";
+const cleanupLabel = "com.buildkite.cleanup";
+
+async function agentToken(): Promise<string> {
+  return (
+    process.env.BUILDKITE_AGENT_TOKEN ||
+    (await sudoRead(config.tokenFile)) ||
+    fail(`no agent token: put it in ${config.tokenFile}`)
+  );
+}
+
+type TartAgentOptions = { release: number; spawn: number };
+
+export async function installTartAgent({ release, spawn }: TartAgentOptions): Promise<void> {
+  const user = config.ciUser;
+  if ((await consoleUser()) !== user)
+    fail(`${user} is not logged in at the console; run setup-user, reboot, then retry`);
+  const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${config.tart.image}`);
+  if (!baked) fail(`no ${config.tart.image} image for ${user}; run bake as ${user} first`);
+
+  const home = await ciUserHome();
+  const uid = await ciUserId();
+  const configPath = join(home, ".buildkite-agent", "buildkite-agent.cfg");
+  const builds = join(home, "tart-builds");
+  const logs = join(home, "Library", "Logs", "buildkite-agent");
+  const hooks = join(config.installDir, "hooks");
+  const owner = `${user}:staff`;
+
+  await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${join(home, ".buildkite-agent")}`;
+
+  await sudoWrite(
+    configPath,
+    [
+      `token="${await agentToken()}"`,
+      `name="%hostname-tart-${release}-%spawn"`,
+      `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`,
+      `hooks-path="${hooks}"`,
+      `build-path="${builds}"`,
+      `spawn=${spawn}`,
+      `cancel-grace-period=30`,
+      "",
+    ].join("\n"),
+    "600",
+    owner,
+  );
+
+  // a LaunchAgent in the auto-login user's GUI session, because Virtualization.framework will not start guests outside one
+  await sudoWrite(
+    `/Library/LaunchAgents/${agentLabel}.plist`,
+    plist({
+      Label: agentLabel,
+      UserName: user,
+      LimitLoadToSessionType: "Aqua",
+      ProcessType: "Interactive",
+      EnvironmentVariables: { PATH: path },
+      ProgramArguments: [config.buildkiteAgent.bin, "start", "--config", configPath],
+      WorkingDirectory: home,
+      RunAtLoad: true,
+      KeepAlive: true,
+      ThrottleInterval: 10,
+      StandardOutPath: join(logs, "buildkite-agent.log"),
+      StandardErrorPath: join(logs, "buildkite-agent.log"),
+    }),
+  );
+
+  const nightly = [
+    `PATH=${path}`,
+    `launchctl bootout gui/${uid}/${agentLabel}`,
+    "sleep 40",
+    `for vm in $(sudo -u ${user} tart list --source local --quiet | grep '^bk-'); do sudo -u ${user} tart delete $vm; done`,
+    `rm -rf ${builds}/* /tmp/bk-*.log`,
+    "shutdown -r now",
+  ].join("; ");
+
+  await sudoWrite(
+    `/Library/LaunchDaemons/${cleanupLabel}.plist`,
+    plist({
+      Label: cleanupLabel,
+      ProgramArguments: ["/bin/sh", "-c", nightly],
+      StartCalendarInterval: { Hour: 5, Minute: 0 },
+    }),
+  );
+
+  await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow();
+  await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow();
+  await unload(`gui/${uid}/${agentLabel}`);
+  await $`sudo launchctl bootstrap gui/${uid} /Library/LaunchAgents/${agentLabel}.plist`;
+  await $`sudo launchctl bootstrap system /Library/LaunchDaemons/${cleanupLabel}.plist`;
+
+  await sleep(4000);
+  console.log(await output($`sudo tail -4 ${join(logs, "buildkite-agent.log")}`));
+}
+
+export async function installBareAgent(): Promise<void> {
+  const checkout = join(process.env.HOME!, "bun-bootstrap");
+  const token = await agentToken();
+  await $`sudo env BUILDKITE_AGENT_TOKEN=${token} PATH=${path} node scripts/agent.mjs install`.cwd(checkout);
+  await sleep(4000);
+  console.log(
+    await output($`tail -4 ${join(process.env.HOME!, "Library", "Logs", "buildkite-agent", "buildkite-agent.log")}`),
+  );
+}
+
+// bootout returns before a busy agent has finished its cancel grace period, and bootstrap fails with EIO until it has
+async function unload(target: string): Promise<void> {
+  await $`sudo launchctl bootout ${target}`.quiet().nothrow();
+  await poll(30, 2000, async () => ((await succeeds($`sudo launchctl print ${target}`)) ? undefined : true));
+}

scripts/darwin-ci/lib/bake.ts

@@ -0,0 +1,50 @@
+import { join } from "node:path";
+import { config, toolchain } from "./config";
+import { ensureHostKey, guest } from "./guest";
+import { fail, log } from "./shell";
+import { tart } from "./tart";
+
+type BakeOptions = { base: string; ref: string };
+
+export async function bake({ base, ref }: BakeOptions): Promise<void> {
+  const { image, cpu, memoryMb } = config.tart;
+  const staging = `${image}-new`;
+  const publicKey = await ensureHostKey();
+
+  log(`pull ${base}`);
+  await tart.pull(base);
+
+  await tart.destroy(staging);
+  log(`clone ${base} -> ${staging} (cpu=${cpu} mem=${memoryMb}MB)`);
+  await tart.clone(base, staging);
+  await tart.configure(staging, cpu, memoryMb);
+
+  log(`boot ${staging}`);
+  const vm = tart.start(staging, `/tmp/bake-${staging}.log`);
+  const ip = (await tart.waitForSsh(staging, 60)) ?? fail(`${staging} never came up; see /tmp/bake-${staging}.log`);
+  log(`guest up at ${ip}`);
+
+  (await tart.waitForAgent(staging)) ?? fail("tart guest agent not responding");
+  await tart.exec(
+    staging,
+    `mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${publicKey}' > ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys`,
+  );
+
+  const g = guest(ip);
+  await g.push(join(import.meta.dir, "..", "guest", "bake.sh"), "/tmp/bake.sh");
+
+  log(`bootstrap toolchain in guest (${config.bun.repo}@${ref})`);
+  const status = await g.run(
+    `/bin/bash -l /tmp/bake.sh ${config.bun.repo} ${ref} ${config.buildkiteAgent.version} '${toolchain.join(" ")}' 2>&1 | tee /tmp/bake.log; grep -qx BAKE_OK /tmp/bake.log`,
+  );
+  if (status !== 0) fail(`bake failed in guest; ${image} untouched, ${staging} left running at ${ip} for inspection`);
+
+  await g.run("sync; sudo shutdown -h now").catch(() => {});
+  log("waiting for guest to power off");
+  await Promise.race([vm.exited, Bun.sleep(180_000)]);
+  await tart.stop(staging);
+
+  await tart.remove(image);
+  await tart.rename(staging, image);
+  log(`${image} ready`);
+}

scripts/darwin-ci/lib/ci-user.ts

@@ -0,0 +1,42 @@
+import { $ } from "bun";
+import { config } from "./config";
+import { fail, output, succeeds, sudoRead, sudoWrite } from "./shell";
+
+const user = config.ciUser;
+const passwordFile = "/var/root/ci-user-password";
+const kcpasswordKey = [0x7d, 0x89, 0x52, 0x23, 0xd2, 0xbc, 0xdd, 0xea, 0xa3, 0xb9, 0x1f];
+
+export const ciUserExists = () => succeeds($`id ${user}`);
+
+export async function ciUserHome(): Promise<string> {
+  const line = await output($`dscl . -read /Users/${user} NFSHomeDirectory`);
+  return line.split(/\s+/)[1] ?? fail(`no home directory for ${user}`);
+}
+
+export const ciUserId = async () => Number(await output($`id -u ${user}`));
+
+export async function ensureCiUser(): Promise<void> {
+  if (await ciUserExists()) return;
+  await $`sudo /bin/sh -c ${`umask 077; openssl rand -hex 24 > ${passwordFile}`}`;
+  const password = (await sudoRead(passwordFile)) ?? fail(`could not read ${passwordFile}`);
+  await $`sudo sysadminctl -addUser ${user} -fullName CI -shell /bin/zsh -home /Users/${user} -password ${password}`.quiet();
+  await $`sudo createhomedir -c -u ${user}`.quiet().nothrow();
+  await $`sudo dseditgroup -o edit -d ${user} -t user admin`.quiet().nothrow();
+}
+
+// loginwindow reads the auto-login password from /etc/kcpassword, XOR'd with a fixed key and padded to 12 bytes
+function kcpassword(password: string): Uint8Array {
+  const bytes = [...Buffer.from(password), 0];
+  while (bytes.length % 12) bytes.push(0);
+  return Uint8Array.from(bytes, (byte, i) => byte ^ kcpasswordKey[i % kcpasswordKey.length]);
+}
+
+export async function enableAutoLogin(): Promise<void> {
+  const password =
+    (await sudoRead(passwordFile)) ??
+    fail(`${passwordFile} missing; cannot configure auto-login for an existing ${user}`);
+  await sudoWrite("/etc/kcpassword", kcpassword(password), "600");
+  await $`sudo defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser ${user}`;
+  await $`sudo defaults write /Library/Preferences/com.apple.loginwindow DisableScreenLockImmediate -bool true`;
+  await $`sudo pmset -a sleep 0 displaysleep 0 disksleep 0 womp 1 autorestart 1`.quiet();
+}

scripts/darwin-ci/lib/config.ts

@@ -0,0 +1,26 @@
+export const config = {
+  installDir: "/usr/local/share/darwin-ci",
+  tokenFile: process.env.DARWIN_CI_TOKEN_FILE ?? "/var/root/buildkite-agent-token",
+  buildkiteAgent: { version: "3.114.0", bin: "/usr/local/bin/buildkite-agent" },
+  queue: "test-darwin",
+  ciUser: "ci",
+  bun: { repo: "https://github.com/oven-sh/bun.git", ref: "main" },
+  tart: {
+    bin: "/opt/homebrew/bin/tart",
+    baseRemote: "ghcr.io/cirruslabs/macos-sequoia-xcode:latest",
+    image: "bun-ci-base",
+    guestUser: "admin",
+    guestRelease: 15,
+    cpu: 8,
+    memoryMb: 24576,
+    spawn: 2,
+  },
+} as const;
+
+export const toolchain = ["bun", "node", "cmake", "ninja", "ccache", "cargo", "go", "clang-21"];
+
+export function releaseTier(release: number): "latest" | "previous" | "oldest" {
+  if (release >= 26) return "latest";
+  if (release >= 14) return "previous";
+  return "oldest";
+}

scripts/darwin-ci/lib/guest.ts

@@ -0,0 +1,60 @@
+import { $ } from "bun";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { config } from "./config";
+
+export const hostKey = join(homedir(), ".ssh", "id_ed25519");
+
+const sshOptions = [
+  "-i",
+  hostKey,
+  "-o",
+  "IdentitiesOnly=yes",
+  "-o",
+  "IdentityAgent=none",
+  "-o",
+  "StrictHostKeyChecking=no",
+  "-o",
+  "UserKnownHostsFile=/dev/null",
+  "-o",
+  "LogLevel=ERROR",
+  "-o",
+  "ServerAliveInterval=30",
+];
+
+export async function ensureHostKey(): Promise<string> {
+  if (!(await Bun.file(hostKey).exists())) {
+    await $`ssh-keygen -q -t ed25519 -N "" -C ${`${process.env.USER}@darwin-ci`} -f ${hostKey}`;
+  }
+  return (await Bun.file(`${hostKey}.pub`).text()).trim();
+}
+
+export function guest(ip: string) {
+  const target = `${config.tart.guestUser}@${ip}`;
+  const rsh = `ssh ${sshOptions.join(" ")}`;
+
+  return {
+    ip,
+
+    run(command: string, forward: string[] = []): Promise<number> {
+      const proc = Bun.spawn(["ssh", ...sshOptions, ...forward, target, command], {
+        stdin: "ignore",
+        stdout: "inherit",
+        stderr: "inherit",
+      });
+      return proc.exited;
+    },
+
+    capture: (command: string) => $`ssh ${sshOptions} ${target} ${command} < /dev/null`.text(),
+
+    push: (local: string, remote: string) => $`scp ${sshOptions} -q ${local} ${target}:${remote}`.quiet(),
+
+    syncTo: (localDir: string, remoteDir: string) =>
+      $`rsync -a --delete -e ${rsh} ${localDir}/ ${target}:${remoteDir}/`.quiet(),
+
+    collectReports: (remoteDir: string, localDir: string) =>
+      $`rsync -a -e ${rsh} --include=*/ --include=*.xml --include=*.junit --exclude=* --prune-empty-dirs ${target}:${remoteDir}/ ${localDir}/`
+        .quiet()
+        .nothrow(),
+  };
+}

scripts/darwin-ci/lib/host.ts

@@ -0,0 +1,101 @@
+import { $ } from "bun";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { config, toolchain } from "./config";
+import { fail, output, poll, sleep, succeeds, sudoWrite } from "./shell";
+
+export const brewPrefix = process.arch === "arm64" ? "/opt/homebrew" : "/usr/local";
+const brew = `${brewPrefix}/bin/brew`;
+const tailscale = `${brewPrefix}/bin/tailscale`;
+
+export async function disableRemoteManagement(): Promise<void> {
+  const kickstart = "/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart";
+  await $`sudo ${kickstart} -deactivate -stop`.quiet().nothrow();
+  await $`sudo launchctl disable system/com.apple.screensharing`.quiet().nothrow();
+  await $`sudo launchctl bootout system/com.apple.screensharing`.quiet().nothrow();
+}
+
+export async function hardenSshd(): Promise<void> {
+  await sudoWrite(
+    "/etc/ssh/sshd_config.d/000-hardening.conf",
+    "PasswordAuthentication no\nKbdInteractiveAuthentication no\nPermitRootLogin no\n",
+  );
+  await $`sudo launchctl kickstart -k system/com.openssh.sshd`.quiet();
+}
+
+export async function setHostname(name: string): Promise<void> {
+  for (const key of ["ComputerName", "LocalHostName", "HostName"]) {
+    await $`sudo scutil --set ${key} ${name}`;
+  }
+  // bootstrap.sh only appends PATH entries to profiles that already exist
+  await $`touch ~/.profile ~/.zshrc ~/.bash_profile`;
+}
+
+export async function brewInstall(formula: string): Promise<void> {
+  const name = formula.split("/").pop()!;
+  if (await succeeds($`${brew} list ${name}`)) return;
+  await $`${brew} install ${formula}`;
+}
+
+export async function joinTailnet(hostname: string, tags: string | undefined): Promise<void> {
+  await brewInstall("tailscale");
+  await $`sudo ${brewPrefix}/bin/tailscaled install-system-daemon`.quiet().nothrow();
+  await sleep(3000);
+  if (await succeeds($`sudo ${tailscale} status`)) return;
+
+  const tagArgs = tags ? [`--advertise-tags=${tags}`] : [];
+  Bun.spawn(["sudo", tailscale, "up", "--ssh", ...tagArgs, `--hostname=${hostname}`], {
+    stdin: "ignore",
+    stdout: "ignore",
+    stderr: "ignore",
+  }).unref();
+
+  const url = await poll(15, 2000, async () => {
+    const status = JSON.parse((await output($`sudo ${tailscale} status --json`)) || "{}");
+    return typeof status.AuthURL === "string" && status.AuthURL ? (status.AuthURL as string) : undefined;
+  });
+  console.log(
+    url
+      ? `    approve this host in the tailnet admin console: ${url}`
+      : "    tailscale up started; run `tailscale status` for the login URL",
+  );
+}
+
+export async function tailnetSummary(): Promise<string> {
+  const line = await output($`sudo ${tailscale} status --self --peers=false`);
+  return line.split("\n")[0] || "not connected";
+}
+
+export async function installBuildkiteAgent(): Promise<void> {
+  const { version, bin } = config.buildkiteAgent;
+  if ((await output($`${bin} --version`)).includes(version)) return;
+  const arch = process.arch === "arm64" ? "arm64" : "amd64";
+  const url = `https://github.com/buildkite/agent/releases/download/v${version}/buildkite-agent-darwin-${arch}-${version}.tar.gz`;
+  const tmp = mkdtempSync(join(tmpdir(), "buildkite-agent-"));
+  await $`curl -fsSL ${url} | tar -xz -C ${tmp}`;
+  await $`sudo mkdir -p /usr/local/bin && sudo install -m 755 ${join(tmp, "buildkite-agent")} ${bin}`;
+  await $`rm -rf ${tmp}`;
+}
+
+export async function installSelf(): Promise<void> {
+  const source = join(import.meta.dir, "..");
+  await $`sudo mkdir -p ${config.installDir}`;
+  await $`sudo rsync -a --delete --chmod=Fa+r,Da+rx ${source}/ ${config.installDir}/`;
+}
+
+export async function bootstrapToolchain(): Promise<void> {
+  const checkout = join(process.env.HOME!, "bun-bootstrap");
+  await $`rm -rf ${checkout}`;
+  await $`git clone -q --depth=1 --branch ${config.bun.ref} ${config.bun.repo} ${checkout}`;
+  await $`./scripts/bootstrap.sh`.cwd(checkout).nothrow();
+  await verifyToolchain();
+}
+
+export async function verifyToolchain(): Promise<void> {
+  const missing: string[] = [];
+  for (const tool of toolchain) {
+    if (!(await succeeds($`bash -lc ${`command -v ${tool}`}`))) missing.push(tool);
+  }
+  if (missing.length) fail(`toolchain incomplete after bootstrap: ${missing.join(", ")}`);
+}

scripts/darwin-ci/lib/launchd.ts

@@ -0,0 +1,24 @@
+export type PlistValue = string | number | boolean | PlistValue[] | { [key: string]: PlistValue };
+
+function escape(text: string): string {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+function value(v: PlistValue): string {
+  if (typeof v === "string") return `<string>${escape(v)}</string>`;
+  if (typeof v === "number") return `<integer>${v}</integer>`;
+  if (typeof v === "boolean") return v ? "<true/>" : "<false/>";
+  if (Array.isArray(v)) return `<array>${v.map(value).join("")}</array>`;
+  return `<dict>${Object.entries(v)
+    .map(([k, inner]) => `<key>${escape(k)}</key>${value(inner)}`)
+    .join("")}</dict>`;
+}
+
+export function plist(entries: { [key: string]: PlistValue }): string {
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">`,
+    `<plist version="1.0">${value(entries)}</plist>`,
+    "",
+  ].join("\n");
+}

scripts/darwin-ci/lib/shell.ts

@@ -0,0 +1,61 @@
+import { $ } from "bun";
+
+export function log(message: string): void {
+  console.log(`${new Date().toTimeString().slice(0, 8)} ${message}`);
+}
+
+export function step(title: string): void {
+  console.log(`\n=== ${title}`);
+}
+
+export function fail(message: string): never {
+  console.error(`error: ${message}`);
+  process.exit(1);
+}
+
+export const sleep = (ms: number) => Bun.sleep(ms);
+
+export async function succeeds(proc: $.ShellPromise): Promise<boolean> {
+  return (await proc.quiet().nothrow()).exitCode === 0;
+}
+
+export async function output(proc: $.ShellPromise): Promise<string> {
+  return (await proc.quiet().nothrow().text()).trim();
+}
+
+export async function poll<T>(
+  attempts: number,
+  intervalMs: number,
+  probe: () => Promise<T | undefined>,
+): Promise<T | undefined> {
+  for (let i = 0; i < attempts; i++) {
+    const value = await probe();
+    if (value !== undefined) return value;
+    await sleep(intervalMs);
+  }
+  return undefined;
+}
+
+export function portOpen(host: string, port: number): Promise<boolean> {
+  return succeeds($`nc -z -w2 ${host} ${port}`);
+}
+
+export async function sudoWrite(
+  path: string,
+  content: string | Uint8Array,
+  mode = "644",
+  owner = "root:wheel",
+): Promise<void> {
+  await $`sudo tee ${path} < ${new Response(content)}`.quiet();
+  await $`sudo chown ${owner} ${path}`.quiet();
+  await $`sudo chmod ${mode} ${path}`.quiet();
+}
+
+export async function sudoRead(path: string): Promise<string | undefined> {
+  return (await output($`sudo cat ${path}`)) || undefined;
+}
+
+export async function consoleUser(): Promise<string | undefined> {
+  const user = await output($`stat -f %Su /dev/console`);
+  return user && user !== "root" ? user : undefined;
+}

scripts/darwin-ci/lib/tart.ts

@@ -0,0 +1,70 @@
+import { $, type Subprocess } from "bun";
+import { mkdirSync, rmdirSync, rmSync } from "node:fs";
+import { config } from "./config";
+import { output, poll, portOpen, sleep, succeeds } from "./shell";
+
+const bin = config.tart.bin;
+const cloneLock = "/tmp/tart-clone.lock.d";
+
+export const tart = {
+  pull: (image: string) => $`${bin} pull ${image}`,
+
+  clone: (from: string, to: string) => $`${bin} clone ${from} ${to}`.quiet(),
+
+  configure: (vm: string, cpu: number, memoryMb: number) =>
+    $`${bin} set ${vm} --cpu ${cpu} --memory ${memoryMb}`.quiet(),
+
+  start(vm: string, logPath: string): Subprocess {
+    const log = Bun.file(logPath);
+    return Bun.spawn([bin, "run", vm, "--no-graphics"], { stdin: "ignore", stdout: log, stderr: log });
+  },
+
+  ip: async (vm: string) => (await output($`${bin} ip ${vm}`)) || undefined,
+
+  exec: (vm: string, script: string) => $`${bin} exec ${vm} /bin/bash -lc ${script}`.quiet(),
+
+  stop: (vm: string) => $`${bin} stop ${vm} --timeout 5`.quiet().nothrow(),
+
+  remove: (vm: string) => $`${bin} delete ${vm}`.quiet().nothrow(),
+
+  rename: (from: string, to: string) => $`${bin} rename ${from} ${to}`.quiet(),
+
+  exists: (vm: string) => succeeds($`${bin} get ${vm}`),
+
+  async destroy(vm: string): Promise<void> {
+    await tart.stop(vm);
+    await tart.remove(vm);
+  },
+
+  // concurrent clones of one image race; macOS has no flock(1)
+  async cloneLocked(from: string, to: string): Promise<void> {
+    for (let waited = 0; ; waited++) {
+      try {
+        mkdirSync(cloneLock);
+        break;
+      } catch {
+        if (waited >= 120) {
+          rmSync(cloneLock, { recursive: true, force: true });
+          waited = 0;
+        }
+        await sleep(1000);
+      }
+    }
+    try {
+      await tart.clone(from, to);
+    } finally {
+      rmdirSync(cloneLock);
+    }
+  },
+
+  waitForSsh(vm: string, attempts = 30): Promise<string | undefined> {
+    return poll(attempts, 4000, async () => {
+      const ip = await tart.ip(vm);
+      return ip && (await portOpen(ip, 22)) ? ip : undefined;
+    });
+  },
+
+  waitForAgent(vm: string): Promise<true | undefined> {
+    return poll(30, 4000, async () => ((await succeeds(tart.exec(vm, "true"))) ? true : undefined));
+  },
+};

scripts/darwin-ci/main.ts

@@ -0,0 +1,126 @@
+#!/usr/bin/env bun
+import { $ } from "bun";
+import { parseArgs } from "node:util";
+import { installBareAgent, installTartAgent } from "./lib/agent";
+import { bake } from "./lib/bake";
+import { ciUserExists, enableAutoLogin, ensureCiUser } from "./lib/ci-user";
+import { config } from "./lib/config";
+import {
+  bootstrapToolchain,
+  brewInstall,
+  disableRemoteManagement,
+  hardenSshd,
+  installBuildkiteAgent,
+  installSelf,
+  joinTailnet,
+  setHostname,
+  tailnetSummary,
+} from "./lib/host";
+import { consoleUser, fail, step } from "./lib/shell";
+
+const usage = `usage:
+  main.ts provision <hostname> <tart|bare> [--tags <tailscale tags>]   converge a freshly imaged host
+  main.ts setup-user                                                    create the auto-login ${config.ciUser} user
+  main.ts bake [--base <image>] [--ref <bun ref>]                       build ${config.tart.image} (run as ${config.ciUser})
+  main.ts install-agent [--release N] [--spawn N]                       write agent config and launchd jobs
+
+provision, setup-user and install-agent need passwordless sudo.`;
+
+const { positionals, values } = parseArgs({
+  allowPositionals: true,
+  options: {
+    base: { type: "string", default: config.tart.baseRemote },
+    ref: { type: "string", default: config.bun.ref },
+    release: { type: "string", default: String(config.tart.guestRelease) },
+    spawn: { type: "string", default: String(config.tart.spawn) },
+    tags: { type: "string" },
+  },
+});
+
+const [subcommand, ...args] = positionals;
+const agentOptions = { release: Number(values.release), spawn: Number(values.spawn) };
+
+switch (subcommand) {
+  case "provision":
+    await provision(args[0] ?? fail(usage), parseMode(args[1]));
+    break;
+  case "setup-user":
+    await setupUser();
+    break;
+  case "bake":
+    await bake({ base: values.base!, ref: values.ref! });
+    break;
+  case "install-agent":
+    await installTartAgent(agentOptions);
+    break;
+  default:
+    fail(usage);
+}
+
+function parseMode(mode: string | undefined): "tart" | "bare" {
+  if (mode === "tart" || mode === "bare") return mode;
+  return fail(usage);
+}
+
+async function setupUser(): Promise<void> {
+  await ensureCiUser();
+  await enableAutoLogin();
+}
+
+async function provision(name: string, mode: "tart" | "bare"): Promise<void> {
+  step("remote management off, sshd key-only");
+  await disableRemoteManagement();
+  await hardenSshd();
+
+  step(`hostname ${name}`);
+  await setHostname(name);
+
+  step("tailscale");
+  await joinTailnet(name, values.tags);
+
+  step(`buildkite-agent ${config.buildkiteAgent.version}`);
+  await installBuildkiteAgent();
+
+  step(`install scripts to ${config.installDir}`);
+  await installSelf();
+
+  if (mode === "tart") await provisionTart();
+  else await provisionBare();
+
+  step("done");
+  console.log(`tailscale: ${await tailnetSummary()}`);
+}
+
+async function provisionTart(): Promise<void> {
+  if (process.arch !== "arm64") fail("tart mode needs Apple Silicon; use bare on Intel");
+  const user = config.ciUser;
+  const main = `${config.installDir}/main.ts`;
+
+  step("tart");
+  await brewInstall("cirruslabs/cli/tart");
+
+  step(`${user} user with auto-login`);
+  const existed = await ciUserExists();
+  await setupUser();
+  if ((await consoleUser()) !== user) {
+    console.log(
+      `${existed ? "" : `created ${user}; `}reboot so ${user} owns the console session, then re-run this command:`,
+    );
+    console.log("  sudo shutdown -r now");
+    return;
+  }
+
+  step(`bake ${config.tart.image} as ${user}`);
+  await $`sudo -u ${user} -H /usr/local/bin/bun ${main} bake --base ${values.base!} --ref ${values.ref!}`;
+
+  step("agent");
+  await installTartAgent(agentOptions);
+}
+
+async function provisionBare(): Promise<void> {
+  step("toolchain (scripts/bootstrap.sh)");
+  await bootstrapToolchain();
+
+  step("agent (scripts/agent.mjs)");
+  await installBareAgent();
+}