#!/bin/sh # Version: 41 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. # If this script does not work on your machine, please open an issue: # https://github.com/oven-sh/bun/issues # If you need to make a change to this script, such as upgrading a dependency, # increment the version comment to indicate that a new image should be built. # Then, on a PR (image bakes are disabled on main): # 1. Put `[build images]` (or `[build linux images]`) in the commit subject # to bake throwaway images and run CI against them. # 2. Once green, change the subject to `[publish images]` and push again to # bake the real `-vN` image tag. # 3. Merge after the publish run finishes so main never waits on a bake. # See "CI image lifecycle" above getBuildImageStep in .buildkite/ci.mjs. pid="$$" print() { echo "$@" } error() { print "error: $@" >&2 if ! [ "$$" = "$pid" ]; then kill -s TERM "$pid" fi exit 1 } execute() { local opts=$- set -x "$@" { local status=$?; set +x "$opts"; } 2> /dev/null if [ "$status" -ne 0 ]; then error "Command failed: $@" fi } execute_sudo() { if [ "$sudo" = "1" ] || [ -z "$can_sudo" ]; then execute "$@" else execute sudo -n "$@" fi } execute_as_user() { sh="$(require sh)" if [ "$sudo" = "1" ] || [ "$can_sudo" = "1" ]; then if [ -f "$(which sudo)" ]; then execute sudo -n -u "$user" "$sh" -lc "$*" elif [ -f "$(which doas)" ]; then execute doas -u "$user" "$sh" -lc "$*" elif [ -f "$(which su)" ]; then execute su -s "$sh" "$user" -lc "$*" else execute "$sh" -lc "$*" fi else execute "$sh" -lc "$*" fi } grant_to_user() { path="$1" if ! [ -f "$path" ] && ! [ -d "$path" ]; then error "Could not find file or directory: \"$path\"" fi chown="$(require chown)" execute_sudo "$chown" -R "$user:$group" "$path" execute_sudo chmod -R 777 "$path" } which() { command -v "$1" } require() { path="$(which "$1")" if ! [ -f "$path" ]; then error "Command \"$1\" is required, but is not installed." fi print "$path" } fetch() { curl="$(which curl)" if [ -f "$curl" ]; then execute "$curl" -fsSL "$1" else wget="$(which wget)" if [ -f "$wget" ]; then execute "$wget" -qO- "$1" else error "Command \"curl\" or \"wget\" is required, but is not installed." fi fi } compare_version() { if [ "$1" = "$2" ]; then print "0" elif [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]; then print "-1" else print "1" fi } create_directory() { path="$1" path_dir="$path" while ! [ -d "$path_dir" ]; do path_dir="$(dirname "$path_dir")" done path_needs_sudo="0" if ! [ -r "$path_dir" ] || ! [ -w "$path_dir" ]; then path_needs_sudo="1" fi mkdir="$(require mkdir)" if [ "$path_needs_sudo" = "1" ]; then execute_sudo "$mkdir" -p "$path" else execute "$mkdir" -p "$path" fi grant_to_user "$path" } create_tmp_directory() { mktemp="$(require mktemp)" path="$(execute "$mktemp" -d)" grant_to_user "$path" print "$path" } create_file() { path="$1" path_dir="$(dirname "$path")" if ! [ -d "$path_dir" ]; then create_directory "$path_dir" fi path_needs_sudo="0" if ! [ -r "$path" ] || ! [ -w "$path" ]; then path_needs_sudo="1" fi if [ "$path_needs_sudo" = "1" ]; then execute_sudo touch "$path" else execute touch "$path" fi content="$2" if [ -n "$content" ]; then append_file "$path" "$content" fi grant_to_user "$path" } append_file() { path="$1" if ! [ -f "$path" ]; then create_file "$path" fi path_needs_sudo="0" if ! [ -r "$path" ] || ! [ -w "$path" ]; then path_needs_sudo="1" fi content="$2" print "$content" | while read -r line; do if ! grep -q "$line" "$path"; then sh="$(require sh)" if [ "$path_needs_sudo" = "1" ]; then execute_sudo "$sh" -c "echo '$line' >> '$path'" else execute "$sh" -c "echo '$line' >> '$path'" fi fi done } download_file() { file_url="$1" file_tmp_dir="$(create_tmp_directory)" file_tmp_path="$file_tmp_dir/$(basename "$file_url")" fetch "$file_url" > "$file_tmp_path" grant_to_user "$file_tmp_path" print "$file_tmp_path" } # path=$(download_and_verify_file URL sha256) download_and_verify_file() { file_url="$1" hash="$2" path=$(download_file "$file_url") execute sh -c 'echo "'"$hash $path"'" | sha256sum -c -' >/dev/null 2>&1 print "$path" } append_to_profile() { content="$1" profiles=".profile .zprofile .bash_profile .bashrc .zshrc .cshrc" for profile in $profiles; do for profile_path in "$current_home/$profile" "$home/$profile"; do if [ "$ci" = "1" ] || [ -f "$profile_path" ]; then append_file "$profile_path" "$content" fi done done } append_to_path() { path="$1" if ! [ -d "$path" ]; then error "Could not find directory: \"$path\"" fi append_to_profile "export PATH=\"$path:\$PATH\"" export PATH="$path:$PATH" } move_to_bin() { exe_path="$1" if ! [ -f "$exe_path" ]; then error "Could not find executable: \"$exe_path\"" fi usr_paths="/usr/bin /usr/local/bin" for usr_path in $usr_paths; do if [ -d "$usr_path" ] && [ -w "$usr_path" ]; then break fi done grant_to_user "$exe_path" execute_sudo mv -f "$exe_path" "$usr_path/$(basename "$exe_path")" } check_features() { print "Checking features..." for arg in "$@"; do case "$arg" in *--ci*) ci=1 print "CI: enabled" ;; *--gcc-13*) gcc_version="13" print "GCC 13: enabled" ;; esac done } check_operating_system() { print "Checking operating system..." uname="$(require uname)" os="$("$uname" -s)" case "$os" in Linux) os="linux" ;; Darwin) os="darwin" ;; *) error "Unsupported operating system: $os" ;; esac print "Operating System: $os" arch="$("$uname" -m)" case "$arch" in x86_64 | x64 | amd64) arch="x64" ;; aarch64 | arm64) arch="aarch64" ;; *) error "Unsupported architecture: $arch" ;; esac print "Architecture: $arch" kernel="$("$uname" -r)" print "Kernel: $kernel" case "$os" in linux) if [ -f "/etc/alpine-release" ]; then distro="alpine" abi="musl" alpine="$(cat /etc/alpine-release)" if [ "$alpine" ~ "_" ]; then release="$(print "$alpine" | cut -d_ -f1)-edge" else release="$alpine" fi elif [ -f "/etc/os-release" ]; then . /etc/os-release if [ -n "$ID" ]; then distro="$ID" fi if [ -n "$VERSION_ID" ]; then release="$VERSION_ID" fi fi ;; darwin) sw_vers="$(which sw_vers)" if [ -f "$sw_vers" ]; then distro="$("$sw_vers" -productName)" release="$("$sw_vers" -productVersion)" fi case "$arch" in x64) sysctl="$(which sysctl)" if [ -f "$sysctl" ] && [ "$("$sysctl" -n sysctl.proc_translated 2>/dev/null)" = "1" ]; then arch="aarch64" rosetta="1" print "Rosetta: enabled" fi ;; esac ;; esac if [ -n "$distro" ]; then print "Distribution: $distro $release" fi case "$os" in linux) ldd="$(which ldd)" if [ -f "$ldd" ]; then ldd_version="$($ldd --version 2>&1)" abi_version="$(print "$ldd_version" | grep -o -E '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n 1)" case "$ldd_version" in *musl*) abi="musl" ;; *GNU* | *GLIBC*) abi="gnu" ;; esac fi if [ -n "$abi" ]; then print "ABI: $abi $abi_version" fi ;; esac } check_inside_docker() { if ! [ "$os" = "linux" ]; then return fi print "Checking if inside Docker..." if [ -f "/.dockerenv" ]; then docker=1 else if [ -f "/proc/1/cgroup" ]; then case "$(cat /proc/1/cgroup)" in */docker/*) docker=1 ;; esac fi if [ -f "/proc/self/mountinfo" ]; then case "$(cat /proc/self/mountinfo)" in */docker/*) docker=1 ;; esac fi fi if [ "$docker" = "1" ]; then print "Docker: enabled" fi } check_package_manager() { print "Checking package manager..." case "$os" in darwin) if ! [ -f "$(which brew)" ]; then install_brew fi pm="brew" ;; linux) if [ -f "$(which apt-get)" ]; then pm="apt" elif [ -f "$(which dnf)" ]; then pm="dnf" elif [ -f "$(which yum)" ]; then pm="yum" elif [ -f "$(which apk)" ]; then pm="apk" else error "No package manager found. (apt, dnf, yum, apk)" fi ;; esac print "Package manager: $pm" print "Updating package manager..." case "$pm" in apt) export DEBIAN_FRONTEND=noninteractive package_manager update -y ;; apk) package_manager update ;; esac } check_user() { print "Checking user..." if [ -n "$SUDO_USER" ]; then user="$SUDO_USER" else id="$(require id)" user="$("$id" -un)" group="$("$id" -gn)" fi if [ -z "$user" ]; then error "Could not determine user" fi print "User: $user" print "Group: $group" home="$(execute_as_user echo '~')" if [ -z "$home" ] || [ "$home" = "~" ]; then error "Could not determine home directory for user: $user" fi print "Home: $home" id="$(which id)" if [ -f "$id" ] && [ "$($id -u)" = "0" ]; then sudo=1 print "Sudo: enabled" elif [ -f "$(which sudo)" ] && [ "$(sudo -n echo 1 2>/dev/null)" = "1" ]; then can_sudo=1 print "Sudo: can be used" fi current_user="$user" current_group="$group" current_home="$home" } check_ulimit() { if ! [ "$os" = "linux" ]; then return fi if ! [ "$ci" = "1" ]; then return fi print "Checking ulimits..." systemd_conf="/etc/systemd/system.conf" limits_conf="/etc/security/limits.d/99-unlimited.conf" create_file "$limits_conf" limits="core data fsize memlock nofile rss stack cpu nproc as locks sigpending msgqueue" for limit in $limits; do limit_upper="$(print "$limit" | tr '[:lower:]' '[:upper:]')" limit_value="unlimited" case "$limit" in nofile | nproc) limit_value="1048576" ;; esac if [ -f "$limits_conf" ]; then limit_users="root *" for limit_user in $limit_users; do append_file "$limits_conf" "$limit_user soft $limit $limit_value" append_file "$limits_conf" "$limit_user hard $limit $limit_value" done fi if [ -f "$systemd_conf" ]; then # in systemd's configuration you need to say "infinity" when you mean "unlimited" if [ "$limit_value" = "unlimited" ]; then limit_value="infinity" fi append_file "$systemd_conf" "DefaultLimit$limit_upper=$limit_value" fi done rc_conf="/etc/rc.conf" if [ -f "$rc_conf" ]; then rc_ulimit="" limit_flags="c d e f i l m n q r s t u v x" for limit_flag in $limit_flags; do limit_value="unlimited" case "$limit_flag" in n | u) limit_value="1048576" ;; esac rc_ulimit="$rc_ulimit -$limit_flag $limit_value" done append_file "$rc_conf" "rc_ulimit=\"$rc_ulimit\"" fi pam_confs="/etc/pam.d/common-session /etc/pam.d/common-session-noninteractive" for pam_conf in $pam_confs; do if [ -f "$pam_conf" ]; then append_file "$pam_conf" "session optional pam_limits.so" fi done systemctl="$(which systemctl)" if [ -f "$systemctl" ]; then execute_sudo "$systemctl" daemon-reload fi # Configure dpkg and apt for faster operation in CI environments if [ "$ci" = "1" ] && [ "$pm" = "apt" ]; then dpkg_conf="/etc/dpkg/dpkg.cfg.d/01-ci-options" execute_sudo create_directory "$(dirname "$dpkg_conf")" append_file "$dpkg_conf" "force-unsafe-io" append_file "$dpkg_conf" "no-debsig" apt_conf="/etc/apt/apt.conf.d/99-ci-options" execute_sudo create_directory "$(dirname "$apt_conf")" append_file "$apt_conf" 'Acquire::Languages "none";' append_file "$apt_conf" 'Acquire::GzipIndexes "true";' append_file "$apt_conf" 'Acquire::CompressionTypes::Order:: "gz";' append_file "$apt_conf" 'APT::Get::Install-Recommends "false";' append_file "$apt_conf" 'APT::Get::Install-Suggests "false";' append_file "$apt_conf" 'Dpkg::Options { "--force-confdef"; "--force-confold"; }' fi } package_manager() { case "$pm" in apt) execute_sudo apt-get "$@" ;; dnf) case "$distro" in rhel) execute_sudo dnf \ --disableplugin=subscription-manager \ "$@" ;; *) execute_sudo dnf "$@" ;; esac ;; yum) execute_sudo yum "$@" ;; apk) execute_sudo apk "$@" ;; brew) execute_as_user brew "$@" ;; pkg) execute_sudo pkg "$@" ;; *) error "Unsupported package manager: $pm" ;; esac } check_package() { case "$pm" in apt) apt-cache policy "$1" ;; dnf | yum | brew) package_manager info "$1" ;; *) error "Unsupported package manager: $pm" ;; esac } install_packages() { case "$pm" in apt) package_manager install \ --yes \ --no-install-recommends \ --fix-missing \ "$@" ;; dnf) package_manager install \ --assumeyes \ --nodocs \ --noautoremove \ --allowerasing \ "$@" ;; yum) package_manager install -y "$@" ;; brew) package_manager install \ --force \ --formula \ "$@" package_manager link \ --force \ --overwrite \ "$@" ;; apk) package_manager add \ --no-cache \ --no-interactive \ --no-progress \ "$@" ;; pkg) package_manager install "$@" ;; *) error "Unsupported package manager: $pm" ;; esac } install_brew() { print "Installing Homebrew..." bash="$(require bash)" script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") execute_as_user "$bash" -lc "NONINTERACTIVE=1 $script" case "$arch" in x64) append_to_path "/usr/local/bin" ;; aarch64) append_to_path "/opt/homebrew/bin" ;; esac case "$ci" in 1) append_to_profile "export HOMEBREW_NO_INSTALL_CLEANUP=1" append_to_profile "export HOMEBREW_NO_AUTO_UPDATE=1" append_to_profile "export HOMEBREW_NO_ANALYTICS=1" ;; esac } install_common_software() { case "$pm" in apt) # software-properties-common is not available in Debian Trixie if [ "$distro" = "debian" ] && [ "$release" = "13" ]; then install_packages \ apt-transport-https else install_packages \ apt-transport-https \ software-properties-common fi # https://packages.debian.org # https://packages.ubuntu.com # lsb-release: apt.llvm.org/llvm.sh greps `lsb_release -cs`; debian # slim images don't ship it (ubuntu cloud images do). install_packages \ bash \ ca-certificates \ curl \ htop \ gnupg \ git \ lsb-release \ unzip \ wget \ libc6-dbg ;; dnf) # https://packages.fedoraproject.org install_packages \ bash \ ca-certificates \ curl \ htop \ gnupg \ git \ unzip \ wget \ dnf-plugins-core ;; apk) # https://pkgs.alpinelinux.org/packages install_packages \ bash \ ca-certificates \ curl \ htop \ gnupg \ git \ unzip \ wget \ ;; pkg) # https://www.freshports.org install_packages \ shells/bash \ ftp/curl \ sysutils/htop \ security/gnupg \ devel/git \ archivers/unzip \ ftp/wget \ editors/vim \ sysutils/neofetch \ ;; esac case "$distro" in amzn | alpine) install_packages \ tar ;; rhel) rhel_version="$(execute rpm -E %rhel)" install_packages \ "https://dl.fedoraproject.org/pub/epel/epel-release-latest-$rhel_version.noarch.rpm" ;; centos) install_packages \ epel-release ;; esac crb="$(which crb)" if [ -f "$crb" ]; then execute "$crb" enable fi install_rosetta install_nodejs install_bun install_curl_h3 install_tailscale install_buildkite } nodejs_version_exact() { print "26.3.0" } nodejs_version() { print "$(nodejs_version_exact)" | cut -d. -f1 } install_nodejs() { # Download Node.js directly from nodejs.org nodejs_version="$(nodejs_version_exact)" # Determine platform name for Node.js download case "$os" in darwin) nodejs_platform="darwin" ;; linux) nodejs_platform="linux" ;; *) error "Unsupported OS for Node.js download: $os" ;; esac # Determine architecture name for Node.js download case "$arch" in x64) nodejs_arch="x64" ;; aarch64) nodejs_arch="arm64" ;; *) error "Unsupported architecture for Node.js download: $arch" ;; esac case "$abi" in musl) # nodejs.org doesn't publish musl binaries; the unofficial-builds # project (nodejs/unofficial-builds) ships both x64-musl and # arm64-musl for current releases. (The old private S3 mirror at # bun-nodejs-release predates arm64-musl being available there.) nodejs_mirror="https://unofficial-builds.nodejs.org/download/release" nodejs_foldername="node-v$nodejs_version-$nodejs_platform-$nodejs_arch-musl" ;; *) nodejs_mirror="https://nodejs.org/dist" nodejs_foldername="node-v$nodejs_version-$nodejs_platform-$nodejs_arch" ;; esac # Download Node.js binary archive nodejs_url="$nodejs_mirror/v$nodejs_version/$nodejs_foldername.tar.gz" nodejs_tar="$(download_file "$nodejs_url")" nodejs_extract_dir="$(dirname "$nodejs_tar")" # Extract Node.js execute tar -xzf "$nodejs_tar" -C "$nodejs_extract_dir" # Install Node.js binaries to system nodejs_dir="$nodejs_extract_dir/$nodejs_foldername" # Copy bin files preserving symlinks for file in "$nodejs_dir/bin/"*; do filename="$(basename "$file")" if [ -L "$file" ]; then # Get the symlink target target="$(readlink "$file")" # The symlinks are relative (like ../lib/node_modules/npm/bin/npm-cli.js) # and will work correctly from /usr/local/bin since we're copying # node_modules to /usr/local/lib/node_modules execute_sudo ln -sf "$target" "/usr/local/bin/$filename" elif [ -f "$file" ]; then # Copy regular files execute_sudo cp -f "$file" "/usr/local/bin/$filename" execute_sudo chmod +x "/usr/local/bin/$filename" fi done # Copy node_modules directory to lib if [ -d "$nodejs_dir/lib/node_modules" ]; then execute_sudo mkdir -p "/usr/local/lib" execute_sudo cp -Rf "$nodejs_dir/lib/node_modules" "/usr/local/lib/" fi # Copy include files if they exist if [ -d "$nodejs_dir/include" ]; then execute_sudo mkdir -p "/usr/local/include" execute_sudo cp -Rf "$nodejs_dir/include/node" "/usr/local/include/" fi # Copy share files if they exist (man pages, etc.) if [ -d "$nodejs_dir/share" ]; then execute_sudo mkdir -p "/usr/local/share" # Copy only node-specific directories for sharedir in "$nodejs_dir/share/"*; do if [ -d "$sharedir" ]; then dirname="$(basename "$sharedir")" execute_sudo cp -Rf "$sharedir" "/usr/local/share/" fi done fi # Ensure /usr/local/bin is in PATH if ! echo "$PATH" | grep -q "/usr/local/bin"; then print "Adding /usr/local/bin to PATH" append_to_profile 'export PATH="/usr/local/bin:$PATH"' export PATH="/usr/local/bin:$PATH" fi # Verify Node.js installation if ! command -v node >/dev/null 2>&1; then error "Node.js installation failed: 'node' command not found in PATH" fi installed_version="$(node --version 2>/dev/null || echo "unknown")" expected_version="v$nodejs_version" if [ "$installed_version" != "$expected_version" ]; then error "Node.js installation failed: expected version $expected_version but got $installed_version. Please check your PATH and try running 'which node' to debug." fi print "Node.js $installed_version installed successfully" # Ensure that Node.js headers are always pre-downloaded so that we don't rely on node-gyp install_nodejs_headers } install_nodejs_headers() { nodejs_version="$(nodejs_version_exact)" nodejs_headers_tar="$(download_file "https://nodejs.org/download/release/v$nodejs_version/node-v$nodejs_version-headers.tar.gz")" nodejs_headers_dir="$(dirname "$nodejs_headers_tar")" execute tar -xzf "$nodejs_headers_tar" -C "$nodejs_headers_dir" nodejs_headers_include="$nodejs_headers_dir/node-v$nodejs_version/include" execute_sudo cp -R "$nodejs_headers_include" "/usr/local/" # Also install to node-gyp cache locations for different node-gyp versions # This ensures node-gyp finds headers without downloading them setup_node_gyp_cache "$nodejs_version" "$nodejs_headers_dir/node-v$nodejs_version" } setup_node_gyp_cache() { nodejs_version="$1" headers_source="$2" cache_dir="$home/.cache/node-gyp/$nodejs_version" create_directory "$cache_dir" # Copy headers if [ -d "$headers_source/include" ]; then cp -R "$headers_source/include" "$cache_dir/" 2>/dev/null || true fi # Create installVersion file (node-gyp expects this) echo "11" > "$cache_dir/installVersion" 2>/dev/null || true # For Linux, we don't need .lib files like Windows # but create the directory structure node-gyp expects case "$arch" in x86_64|amd64) create_directory "$cache_dir/lib/x64" 2>/dev/null || true ;; aarch64|arm64) create_directory "$cache_dir/lib/arm64" 2>/dev/null || true ;; *) create_directory "$cache_dir/lib" 2>/dev/null || true ;; esac # Ensure entire path is accessible, not just last component grant_to_user "$home/.cache" } bun_version_exact() { print "1.3.13" } curl_h3_version() { # https://github.com/stunnel/static-curl/releases print "8.19.0" } # Installs a fully-static curl built with nghttp3/ngtcp2 as `curl-h3` so the # HTTP/3 server tests (test/js/bun/http/serve-http3.test.ts, fetch-h3.ts) can # run in CI. Kept separate from the system `curl` so nothing else changes # behavior. Tests discover it via $CURL_HTTP3, then `curl-h3` in PATH. install_curl_h3() { case "$arch" in x64) curl_h3_arch="x86_64" ;; aarch64) curl_h3_arch="aarch64" ;; *) return ;; esac case "$os" in linux) case "$abi" in musl) curl_h3_asset="curl-linux-$curl_h3_arch-musl" ;; *) curl_h3_asset="curl-linux-$curl_h3_arch-glibc" ;; esac ;; darwin) case "$arch" in aarch64) curl_h3_asset="curl-macos-arm64" ;; *) curl_h3_asset="curl-macos-x86_64" ;; esac ;; *) return ;; esac case "$pm" in apt) install_packages xz-utils ;; apk | dnf | yum | zypper) install_packages xz ;; esac curl_h3_url="https://github.com/stunnel/static-curl/releases/download/$(curl_h3_version)/$curl_h3_asset-$(curl_h3_version).tar.xz" curl_h3_tar="$(download_file "$curl_h3_url")" curl_h3_dir="$(dirname "$curl_h3_tar")" execute tar -xJf "$curl_h3_tar" -C "$curl_h3_dir" curl execute mv "$curl_h3_dir/curl" "$curl_h3_dir/curl-h3" move_to_bin "$curl_h3_dir/curl-h3" curl_h3_bin="$(which curl-h3)" append_to_profile "export CURL_HTTP3=$curl_h3_bin" execute "$curl_h3_bin" --version | head -n1 } install_bun() { install_packages unzip case "$pm" in apk) install_packages \ libgcc \ libstdc++ ;; esac case "$abi" in musl) bun_triplet="bun-$os-$arch-$abi" ;; *) bun_triplet="bun-$os-$arch" ;; esac unzip="$(require unzip)" bun_download_url="https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v$(bun_version_exact)/$bun_triplet.zip" bun_zip="$(download_file "$bun_download_url")" bun_tmpdir="$(dirname "$bun_zip")" execute "$unzip" -o "$bun_zip" -d "$bun_tmpdir" move_to_bin "$bun_tmpdir/$bun_triplet/bun" bun_path="$(require bun)" execute_sudo ln -sf "$bun_path" "$(dirname "$bun_path")/bunx" } install_cmake() { case "$os-$pm" in darwin-* | linux-apk) install_packages cmake ;; linux-*) sh="$(require sh)" cmake_version="3.30.5" case "$arch" in x64) cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh" ;; aarch64) cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh" ;; esac cmake_script=$(download_file "$cmake_url") execute_sudo "$sh" "$cmake_script" \ --skip-license \ --prefix=/usr ;; esac } install_rosetta() { case "$os" in darwin) if ! [ "$(which arch)" ]; then execute softwareupdate \ --install-rosetta \ --agree-to-license fi ;; esac } install_build_essentials() { case "$pm" in apt) install_packages \ build-essential \ ninja-build \ xz-utils \ pkg-config \ golang install_packages apache2-utils # QEMU user-mode for baseline CPU verification in CI (parity with # the apk arm below; debian ships all target arches in one package). install_packages qemu-user ;; dnf | yum) install_packages \ gcc-c++ \ xz \ pkg-config \ golang case "$distro" in rhel) ;; *) install_packages ninja-build ;; esac ;; brew) install_packages \ ninja \ pkg-config \ golang ;; apk) install_packages \ build-base \ linux-headers \ ninja \ go \ xz install_packages apache2-utils # QEMU user-mode for baseline CPU verification in CI case "$arch" in x64) install_packages qemu-x86_64 ;; aarch64) install_packages qemu-aarch64 ;; esac ;; esac case "$distro-$pm" in amzn-dnf) package_manager groupinstall -y "Development Tools" ;; esac case "$os" in linux) install_packages \ make \ nasm \ python3 \ libtool \ ruby \ perl \ ;; esac install_cmake install_llvm install_gcc install_rust # Cross-compile sysroots + runtimes are only needed on the single build # host (buildHostPlatform in .buildkite/ci.mjs); test images never # cross-compile, so skip the ~3GB of NDK/SDK/sysroot downloads there. if is_ci_build_host; then install_cross_compiler_rt install_android_ndk install_freebsd_sysroot install_linux_glibc_sysroot install_linux_musl_sysroot install_windows_sysroot install_macos_sdk fi install_ccache install_docker } is_ci_build_host() { # Must match buildHostPlatform in .buildkite/ci.mjs. [ "$os-$distro-$arch-$ci" = "linux-debian-aarch64-1" ] } llvm_version_exact() { print "21.1.8" } llvm_version() { print "$(llvm_version_exact)" | cut -d. -f1 } install_llvm() { case "$pm" in apt) # apt.llvm.org's GPG key uses SHA1, which Debian 13+ (sqv) rejects since 2026-02-01. # Override the sequoia crypto policy to extend the SHA1 deadline. # See: https://github.com/llvm/llvm-project/issues/153385 if [ -x /usr/bin/sqv ] && [ -f /usr/share/apt/default-sequoia.config ]; then execute_sudo mkdir -p /etc/crypto-policies/back-ends execute_sudo /usr/bin/sh -c "sed 's/sha1.second_preimage_resistance = 2026-02-01/sha1.second_preimage_resistance = 2028-02-01/' /usr/share/apt/default-sequoia.config > /etc/crypto-policies/back-ends/apt-sequoia.config" fi bash="$(require bash)" llvm_script="$(download_file "https://apt.llvm.org/llvm.sh")" execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all # Install llvm-symbolizer explicitly to ensure it's available for ASAN install_packages "llvm-$(llvm_version)-tools" # Put the full LLVM bin dir on PATH so unversioned llvm-objcopy, # llvm-strip, llvm-ar etc. resolve (debian only symlinks a subset). append_to_path "/usr/lib/llvm-$(llvm_version)/bin" ;; brew) install_packages "llvm@$(llvm_version)" ;; apk) install_packages \ "llvm$(llvm_version)" \ "clang$(llvm_version)" \ "scudo-malloc" \ "lld$(llvm_version)" \ "llvm$(llvm_version)-dev" # Ensures llvm-symbolizer is installed ;; esac } install_cross_compiler_rt() { # x64 asan cross needs amd64 compiler-rt. Best-effort: raw calls, never # abort (apt.llvm.org may not carry amd64 on every arm64 repo). if [ "$sudo" = "1" ] || [ -z "$can_sudo" ]; then _s=""; else _s="sudo -n"; fi $_s dpkg --add-architecture amd64 || return $_s apt-get update -qq || return $_s apt-get install --yes --no-install-recommends \ "libclang-rt-$(llvm_version)-dev:amd64" || true } install_gcc() { if ! [ "$os" = "linux" ] || ! [ "$distro" = "ubuntu" ] || [ -z "$gcc_version" ]; then return fi # Taken from WebKit's Dockerfile. # https://github.com/oven-sh/WebKit/blob/816a3c02e0f8b53f8eec06b5ed911192589b51e2/Dockerfile execute_sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y execute_sudo apt update -y execute_sudo apt install -y \ "gcc-$gcc_version" \ "g++-$gcc_version" \ "libgcc-$gcc_version-dev" \ "libstdc++-$gcc_version-dev" \ libasan6 \ libubsan1 \ libatomic1 \ libtsan0 \ liblsan0 \ libgfortran5 \ libc6-dev execute_sudo update-alternatives \ --install /usr/bin/gcc gcc "/usr/bin/gcc-$gcc_version" 130 \ --slave /usr/bin/g++ g++ "/usr/bin/g++-$gcc_version" \ --slave /usr/bin/gcc-ar gcc-ar "/usr/bin/gcc-ar-$gcc_version" \ --slave /usr/bin/gcc-nm gcc-nm "/usr/bin/gcc-nm-$gcc_version" \ --slave /usr/bin/gcc-ranlib gcc-ranlib "/usr/bin/gcc-ranlib-$gcc_version" case "$arch" in x64) arch_path="x86_64-linux-gnu" ;; aarch64) arch_path="aarch64-linux-gnu" ;; esac llvm_v="21" append_to_profile "export CC=clang-${llvm_v}" append_to_profile "export CXX=clang++-${llvm_v}" append_to_profile "export AR=llvm-ar-${llvm_v}" append_to_profile "export RANLIB=llvm-ranlib-${llvm_v}" append_to_profile "export LD=lld-${llvm_v}" append_to_profile "export LD_LIBRARY_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}:/usr/lib/${arch_path}" append_to_profile "export LIBRARY_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}:/usr/lib/${arch_path}" append_to_profile "export CPLUS_INCLUDE_PATH=/usr/include/c++/${gcc_version}:/usr/include/${arch_path}/c++/${gcc_version}" append_to_profile "export C_INCLUDE_PATH=/usr/lib/gcc/${arch_path}/${gcc_version}/include" gcc_path="/usr/lib/gcc/$arch_path/$gcc_version" create_directory "$gcc_path" execute_sudo ln -sf /usr/lib/$arch_path/libstdc++.so.6 "$gcc_path/libstdc++.so.6" ld_conf_path="/etc/ld.so.conf.d/gcc-$gcc_version.conf" append_file "$ld_conf_path" "$gcc_path" append_file "$ld_conf_path" "/usr/lib/$arch_path" execute_sudo ldconfig execute_sudo ln -sf $(which clang-$llvm_v) /usr/bin/clang execute_sudo ln -sf $(which clang++-$llvm_v) /usr/bin/clang++ execute_sudo ln -sf $(which lld-$llvm_v) /usr/bin/lld execute_sudo ln -sf $(which lldb-$llvm_v) /usr/bin/lldb execute_sudo ln -sf $(which clangd-$llvm_v) /usr/bin/clangd execute_sudo ln -sf $(which llvm-ar-$llvm_v) /usr/bin/llvm-ar execute_sudo ln -sf $(which ld.lld-$llvm_v) /usr/bin/ld execute_sudo ln -sf $(which clang) /usr/bin/cc execute_sudo ln -sf $(which clang++) /usr/bin/c++ # Make sure llvm-symbolizer is available for ASAN execute_sudo ln -sf $(which llvm-symbolizer-$llvm_v) /usr/bin/llvm-symbolizer } install_ccache() { case "$pm" in apt) install_packages ccache ;; brew) install_packages ccache ;; apk) install_packages ccache ;; dnf|yum) install_packages ccache ;; zypper) install_packages ccache ;; esac } install_rust() { rust_home="/opt/rust" create_directory "$rust_home" append_to_profile "export RUSTUP_HOME=$rust_home" append_to_profile "export CARGO_HOME=$rust_home" sh="$(require sh)" rustup_script=$(download_file "https://sh.rustup.rs") execute "$sh" -lc "$rustup_script -y --no-modify-path" append_to_path "$rust_home/bin" # Ensure all rustup files are accessible (for CI builds where different users run builds) grant_to_user "$rust_home" case "$os" in linux) rustup="$rust_home/bin/rustup" if ! [ -x "$rustup" ]; then error "rustup not found at $rustup after install" fi execute_as_user "$rustup" target add aarch64-linux-android execute_as_user "$rustup" target add x86_64-linux-android # x86_64-unknown-freebsd is Tier 2 (prebuilt std). aarch64 is Tier 3 # (no prebuilt) — lolhtml.ts uses -Zbuild-std for that. execute_as_user "$rustup" target add x86_64-unknown-freebsd # macOS cross-compile lanes build libbun_rust.a for darwin on the # shared Linux rust box (Tier 2, prebuilt std). The ninja rule # self-heals with `rustup target add` if these are missing, but # preinstalling keeps that step off the network. execute_as_user "$rustup" target add aarch64-apple-darwin execute_as_user "$rustup" target add x86_64-apple-darwin # Windows cross-compile targets (--os=windows from a linux host). execute_as_user "$rustup" target add x86_64-pc-windows-msvc execute_as_user "$rustup" target add aarch64-pc-windows-msvc # Linux cross-arch/cross-abi targets so an arm64 glibc host can # cargo-build all four linux triples (x64/aarch64 × gnu/musl). execute_as_user "$rustup" target add x86_64-unknown-linux-gnu execute_as_user "$rustup" target add aarch64-unknown-linux-gnu execute_as_user "$rustup" target add x86_64-unknown-linux-musl execute_as_user "$rustup" target add aarch64-unknown-linux-musl # rust-src for -Zbuild-std (Tier 3 targets without prebuilt std). execute_as_user "$rustup" component add rust-src ;; esac } android_ndk_version() { print "r27c" } install_android_ndk() { case "$os" in linux) ;; *) return ;; esac ndk_version="$(android_ndk_version)" ndk_home="/opt/android-ndk" if ! [ -d "$ndk_home" ]; then ndk_zip=$(download_file "https://dl.google.com/android/repository/android-ndk-${ndk_version}-linux.zip") unzip="$(require unzip)" execute_sudo "$unzip" -q "$ndk_zip" -d /opt execute_sudo mv "/opt/android-ndk-${ndk_version}" "$ndk_home" # Trim ~1.1GB unused (NDK clang/lld, lldb, non-android runtimes). ndk_prebuilt="$ndk_home/toolchains/llvm/prebuilt/linux-x86_64" execute_sudo rm -rf "$ndk_prebuilt/bin" "$ndk_prebuilt/python3" "$ndk_prebuilt/lib/liblldb.so" \ "$ndk_home/simpleperf" "$ndk_home/shader-tools" "$ndk_home/sources" append_to_profile "export ANDROID_NDK_ROOT=$ndk_home" fi # Symlink NDK compiler-rt builtins + libunwind into host clang's resource # dir. clang's driver hardcodes /lib//libclang_rt.* # with no -L fallback, so the file must exist there for any android link. # Done here (as root) so the build user doesn't need write access to /usr. clang="$(which clang-$(llvm_version) || which clang)" if [ -x "$clang" ]; then res_dir="$("$clang" -print-resource-dir)" ndk_clang_ver="$(ls "$ndk_home/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/" | head -1)" ndk_rt="$ndk_home/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/$ndk_clang_ver/lib/linux" execute_sudo mkdir -p "$res_dir/lib/linux" for ndk_arch in aarch64 x86_64; do # Old-style flat layout (apt.llvm.org clang) AND new-style per-triple. execute_sudo ln -sf "$ndk_rt/libclang_rt.builtins-${ndk_arch}-android.a" "$res_dir/lib/linux/" execute_sudo mkdir -p "$res_dir/lib/linux/${ndk_arch}" execute_sudo ln -sf "$ndk_rt/${ndk_arch}/libunwind.a" "$res_dir/lib/linux/${ndk_arch}/" triple_dir="$res_dir/lib/${ndk_arch}-unknown-linux-android28" execute_sudo mkdir -p "$triple_dir" execute_sudo ln -sf "$ndk_rt/libclang_rt.builtins-${ndk_arch}-android.a" "$triple_dir/libclang_rt.builtins.a" execute_sudo ln -sf "$ndk_rt/${ndk_arch}/libunwind.a" "$triple_dir/libunwind.a" done fi } freebsd_version() { print "14.3" } install_freebsd_sysroot() { case "$os" in linux) ;; *) return ;; esac freebsd_ver="$(freebsd_version)" for fbsd_arch in amd64 arm64; do case "$fbsd_arch" in amd64) sysroot="/opt/freebsd-sysroot" ;; arm64) sysroot="/opt/freebsd-sysroot-arm64" ;; esac # Same sentinel detectFreebsdSysroot() uses, plus a /lib file so a # half-extracted (interrupted) sysroot isn't treated as complete. if [ -f "$sysroot/usr/include/sys/param.h" ] && [ -f "$sysroot/lib/libc.so.7" ]; then continue fi execute_sudo rm -rf "$sysroot" execute_sudo mkdir -p "$sysroot" base_txz=$(download_file "https://download.freebsd.org/releases/${fbsd_arch}/${freebsd_ver}-RELEASE/base.txz") execute_sudo tar -C "$sysroot" -xJf "$base_txz" ./usr/include ./usr/lib ./lib done # No FREEBSD_SYSROOT export — detectFreebsdSysroot() picks the # arch-appropriate /opt/freebsd-sysroot{,-arm64} by well-known path. } install_linux_glibc_sysroot() { # ubuntu:20.04 (glibc 2.31) + gcc-13 libstdc++, matching the environment # the prebuilt WebKit is compiled in (see oven-sh/WebKit Dockerfile). All # linux-gnu lanes pass --sysroot pointing here so symbol versions never # exceed 2.31; the --wrap list in flags.ts covers the 2.31 -> 2.17 tail. case "$os-$ci" in linux-1) ;; *) return ;; esac if [ "$abi" = "musl" ]; then return fi if ! [ -f "$(which skopeo)" ]; then install_packages skopeo; fi if ! [ -f "$(which jq)" ]; then install_packages jq; fi # Cross-arch GNU strip for -R .eh_frame (host strip rejects foreign-arch ELF). case "$arch" in aarch64) install_packages binutils-x86-64-linux-gnu ;; x64) install_packages binutils-aarch64-linux-gnu ;; esac skopeo="$(require skopeo)" jq_bin="$(require jq)" if [ "$sudo" = "1" ] || [ -z "$can_sudo" ]; then _s=""; else _s="sudo -n"; fi for sr_arch in x86_64 aarch64; do case "$sr_arch" in x86_64) sysroot="/opt/linux-sysroot-glibc" deb_arch="amd64" apt_base="http://archive.ubuntu.com/ubuntu" ;; aarch64) sysroot="/opt/linux-sysroot-glibc-arm64" deb_arch="arm64" apt_base="http://ports.ubuntu.com/ubuntu-ports" ;; esac if [ -f "$sysroot/usr/include/features.h" ] && [ -d "$sysroot/usr/include/c++/13" ]; then continue fi execute_sudo rm -rf "$sysroot" execute_sudo mkdir -p "$sysroot" tmp="$(create_tmp_directory)" mkdir -p "$tmp/img" # 1. ubuntu:20.04 rootfs (glibc 2.31 runtime libs). execute "$skopeo" copy --override-arch "$deb_arch" \ "docker://docker.io/library/ubuntu:20.04" "dir:$tmp/img" for d in $("$jq_bin" -r '.layers[].digest' "$tmp/img/manifest.json" | sed 's/^sha256://'); do # Raw tar (not execute_sudo) so mknod failures on /dev nodes don't # abort; the libc6 deb below provides the files we actually need. $_s tar -xzf "$tmp/img/$d" -C "$sysroot" 2>/dev/null || true done # 2. focal runtime + dev headers (the minimal base image has runtime # libc but tar may not preserve all symlinks; dev headers are absent). pkgz1=$(download_file "$apt_base/dists/focal-updates/main/binary-${deb_arch}/Packages.gz") pkgz2=$(download_file "$apt_base/dists/focal/main/binary-${deb_arch}/Packages.gz") # focal-updates first so awk's first-match picks the patched version. execute sh -c "gzip -dc '$pkgz1' '$pkgz2' > '$tmp/Packages'" for pkg in libc6 libc6-dev linux-libc-dev libcrypt1 libcrypt-dev; do path=$(awk -v p="$pkg" '$1=="Package:"&&$2==p{f=1} f&&$1=="Filename:"{print $2; exit}' "$tmp/Packages") if [ -n "$path" ]; then deb=$(download_file "$apt_base/$path") execute_sudo dpkg-deb -x "$deb" "$sysroot" fi done # Absolute symlinks from the .debs point at host paths; rewrite them # to stay inside the sysroot so -lpthread/-ldl resolve to target libs. find "$sysroot" -type l 2>/dev/null | while read -r l; do t="$(readlink "$l")" case "$t" in /*) $_s ln -sfn "$sysroot$t" "$l" ;; esac done # libc.so linker script uses absolute /lib// paths; make sure # those resolve inside the sysroot (usrmerge-style symlink if missing). triple="${sr_arch}-linux-gnu" if ! [ -e "$sysroot/lib/$triple/libc.so.6" ]; then execute_sudo mkdir -p "$sysroot/lib" execute_sudo ln -sfn "../usr/lib/$triple" "$sysroot/lib/$triple" fi if [ "$sr_arch" = "x86_64" ] && ! [ -e "$sysroot/lib64" ]; then execute_sudo ln -sfn "usr/lib/$triple" "$sysroot/lib64" fi # 3. gcc-13 (libstdc++-13-dev, libgcc-13-dev) from the same mirrored # release the WebKit Dockerfile uses. gcc13=$(download_file "https://github.com/oven-sh/WebKit/releases/download/gcc-13-focal-debs/gcc-13-focal-${deb_arch}.tar.gz") mkdir -p "$tmp/gcc13" execute tar -xzf "$gcc13" -C "$tmp/gcc13" for deb in "$tmp/gcc13"/*.deb; do execute_sudo dpkg-deb -x "$deb" "$sysroot" done execute_sudo rm -rf "$tmp" if ! [ -d "$sysroot/usr/include/c++/13" ]; then error "$sysroot missing usr/include/c++/13 after gcc-13 overlay" fi print "installed: $sysroot (ubuntu:20.04 glibc 2.31 + gcc-13 libstdc++)" done } alpine_sysroot_version() { # Keep in sync with the alpine release testPlatforms runs on. print "3.23" } install_linux_musl_sysroot() { # musl + modern libstdc++ for --abi=musl cross-compiles from a glibc host. # Populated from alpine's own packages via apk.static so the libstdc++ is # the same one the native alpine test image uses. CI-only. case "$os-$ci" in linux-1) ;; *) return ;; esac if [ "$abi" = "musl" ]; then return fi alpine_ver="$(alpine_sysroot_version)" cdn="https://dl-cdn.alpinelinux.org/alpine/v${alpine_ver}" host_m="$(uname -m)" # apk.static (host arch) can install foreign-arch packages into any root. apk_tmp="$(create_tmp_directory)" idx=$(download_file "$cdn/main/$host_m/APKINDEX.tar.gz") apk_ver="$(tar -xzOf "$idx" APKINDEX 2>/dev/null | awk '/^P:apk-tools-static$/{f=1} f&&/^V:/{print substr($0,3); exit}')" if [ -z "$apk_ver" ]; then error "could not resolve apk-tools-static version from $cdn/main/$host_m/APKINDEX.tar.gz" fi apk_pkg=$(download_file "$cdn/main/$host_m/apk-tools-static-${apk_ver}.apk") execute tar -xzf "$apk_pkg" -C "$apk_tmp" sbin/apk.static apk="$apk_tmp/sbin/apk.static" for ml_arch in x86_64 aarch64; do case "$ml_arch" in x86_64) sysroot="/opt/linux-sysroot-musl" ;; aarch64) sysroot="/opt/linux-sysroot-musl-arm64" ;; esac # Same sentinel detectLinuxMuslSysroot() uses. if [ -f "$sysroot/usr/lib/libc.so" ]; then continue fi execute_sudo rm -rf "$sysroot" execute_sudo mkdir -p "$sysroot" execute_sudo "$apk" --arch "$ml_arch" --root "$sysroot" \ --repository "$cdn/main" --allow-untrusted --no-cache --initdb \ add musl-dev libc-dev linux-headers g++ libstdc++-dev if ! [ -f "$sysroot/usr/lib/libc.so" ]; then error "$sysroot not populated (required for linux-musl cross-arch builds)" fi done execute_sudo rm -rf "$apk_tmp" # No LINUX_MUSL_SYSROOT export: detectLinuxMuslSysroot() picks the # arch-appropriate /opt/linux-sysroot-musl{,-arm64} by well-known path. } xwin_version() { # Keep in sync with XWIN_VERSION in scripts/build/winsysroot.ts and # .buildkite/Dockerfile. print "0.9.0" } install_windows_sysroot() { case "$os" in linux) ;; *) return ;; esac # MSVC CRT/STL + Windows SDK splat for --os=windows cross-compiles, # laid out like a Visual Studio install so clang-cl/lld-link's # /winsysroot flag works (see scripts/build/config.ts `winsysroot`). # Fetched with xwin, which downloads the components from Microsoft's CDN; # --accept-license accepts the Microsoft Software License Terms for the # Build Tools/SDK on behalf of this machine (same terms the Windows CI # images accept when installing VS Build Tools). Machines that skip this # step still work: configure fetches the same splat at build time when # none is present (scripts/build/winsysroot.ts). sysroot="/opt/winsysroot" # Same sentinel scripts/build/winsysroot.ts isCompleteWindowsSysroot() # uses: the SDK lib tree plus a kernel32 import lib plus the ATL headers # (--include-atl), so a half-splatted or pre-ATL sysroot isn't treated as # complete. xwin writes the SDK dirs/files lowercase; a copied VS install # is title-case — accept both. if ls "$sysroot/Windows Kits/10/"[Ll]ib/*/um/x64/kernel32.[Ll]ib >/dev/null 2>&1 && ls "$sysroot"/VC/Tools/MSVC/*/include/atlstr.h >/dev/null 2>&1; then return fi xwin_ver="$(xwin_version)" case "$arch" in aarch64) xwin_triple="aarch64-unknown-linux-musl" ;; *) xwin_triple="x86_64-unknown-linux-musl" ;; esac xwin_tar=$(download_file "https://github.com/Jake-Shadle/xwin/releases/download/${xwin_ver}/xwin-${xwin_ver}-${xwin_triple}.tar.gz") xwin_dir="$(dirname "$xwin_tar")/xwin-extract" execute mkdir -p "$xwin_dir" execute tar -xzf "$xwin_tar" -C "$xwin_dir" --strip-components=1 execute_sudo rm -rf "$sysroot" execute_sudo mkdir -p "$sysroot" # The cache must live on the same filesystem as the output: splat moves # unpacked files with rename(2), which fails with EXDEV (cross-device # link) when the download dir is on tmpfs and /opt is not. xwin_cache="$sysroot.cache" execute_sudo rm -rf "$xwin_cache" execute_sudo mkdir -p "$xwin_cache" # Both target arches in one splat; --include-debug-libs so /MTd (debug # CRT) links work; --include-atl for (rescle.cpp); # winsysroot-style + MS arch notation so clang-cl and lld-link resolve it # with a single /winsysroot flag; symlinks stay ON (default) to fix # include/lib casing on a case-sensitive filesystem. # stdout is dropped: xwin draws progress bars there even without a TTY, # which floods the image-build log. Errors stay on stderr. execute_sudo "$xwin_dir/xwin" --accept-license --arch x86_64,aarch64 --sdk-version 10.0.26100 --crt-version 14.44.17.14 --include-atl --cache-dir "$xwin_cache" \ splat --use-winsysroot-style --preserve-ms-arch-notation --include-debug-libs \ --output "$sysroot" >/dev/null # clang-cl/lld-link compose SDK paths as "Include"/"Lib" (title case); # the winsysroot-style splat writes lowercase — alias both spellings. execute_sudo ln -s include "$sysroot/Windows Kits/10/Include" execute_sudo ln -s lib "$sysroot/Windows Kits/10/Lib" execute_sudo rm -rf "$xwin_dir" "$xwin_cache" # No WINDOWS_SYSROOT export — detectWindowsSysroot() picks up # /opt/winsysroot by well-known path. } install_docker() { case "$pm" in brew) if ! [ -d "/Applications/Docker.app" ]; then package_manager install docker --cask fi ;; pkg) install_packages \ sysutils/docker \ sysutils/docker-compose \ ;; *) case "$distro-$release" in amzn-2 | amzn-1) execute_sudo amazon-linux-extras install docker ;; amzn-* | alpine-*) install_packages docker docker-cli-compose ;; *) sh="$(require sh)" script=$(download_file "https://get.docker.com") execute "$sh" "$script" ;; esac ;; esac systemctl="$(which systemctl)" if [ -f "$systemctl" ]; then execute_sudo "$systemctl" enable docker fi if [ "$os" = "linux" ] && [ "$distro" = "alpine" ]; then execute doas rc-update add docker default execute doas rc-service docker start fi getent="$(which getent)" if [ -n "$("$getent" group docker)" ]; then usermod="$(which usermod)" if [ -z "$usermod" ]; then usermod="$(sudo which usermod)" fi if [ -f "$usermod" ]; then execute_sudo "$usermod" -aG docker "$user" fi fi } macos_sdk_pinned_version() { # Keep in sync with MACOS_SDK_VERSION in scripts/build/macos-sdk.ts. print "26.5" } macos_sdk_clt_release() { # Keep in sync with MACOS_SDK_CLT_RELEASE in scripts/build/macos-sdk.ts. print "26.5" } install_macos_sdk() { # macOS SDK for cross-compiling darwin from this Linux host. Fetched via the # repo's vendored xmac.mjs (pulled at BUN_BOOTSTRAP_REPO_REF) so the same # Apple-CDN download path is used here and at build-time. resolveMacosSdkPath() # in scripts/build/macos-sdk.ts checks /opt/macos-sdk before falling back to # a per-job download. case "$os-$ci" in linux-1) ;; *) return ;; esac # darwin cross lanes never run on a musl host; alpine test images skip it. if [ "$abi" = "musl" ]; then return fi sdk_ver="$(macos_sdk_pinned_version)" sysroot="/opt/macos-sdk" # Same completeness sentinel isMacosSdk() uses. if [ -f "$sysroot/MacOSX${sdk_ver}.sdk/usr/include/sys/syscall.h" ]; then return fi bun_path="$(require bun)" # xmac calls out to `xz` for the pbzx/xip payload. case "$pm" in apt) install_packages xz-utils ;; *) install_packages xz ;; esac repo_ref="${BUN_BOOTSTRAP_REPO_REF:-main}" xmac_mjs=$(download_file "https://raw.githubusercontent.com/oven-sh/bun/${repo_ref}/scripts/build/xmac.mjs") staging="$(create_tmp_directory)" execute_sudo rm -rf "$sysroot" execute_sudo mkdir -p "$sysroot" # stdout is dropped: xmac draws progress bars there even without a TTY. execute "$bun_path" "$xmac_mjs" splat --accept-license --sdk-only \ --release "$(macos_sdk_clt_release)" --sdk "$sdk_ver" \ --output "$staging" --cache-dir "$staging/cache" >/dev/null execute_sudo mv "$staging/SDKs/MacOSX${sdk_ver}.sdk" "$sysroot/" execute_sudo rm -rf "$staging" grant_to_user "$sysroot" } install_tailscale() { if [ "$docker" = "1" ]; then return fi case "$os" in linux) sh="$(require sh)" tailscale_script=$(download_file "https://tailscale.com/install.sh") execute "$sh" "$tailscale_script" ;; darwin) # Homebrew-managed tailscale: a single upgrade path (`brew upgrade`), # proper version reporting (the go-install build shows ERR-BuildInfo), # and `install-system-daemon` writes the launchd plist for us. install_packages tailscale brew_prefix="$(execute_as_user brew --prefix)" execute_sudo "$brew_prefix/bin/tailscaled" install-system-daemon ;; esac } install_fuse_python() { if ! [ "$os" = "linux" ]; then return fi # only linux needs this case "$pm" in apk) # Build and install from source (https://github.com/libfuse/python-fuse/blob/master/INSTALL) install_packages \ python3-dev \ fuse-dev \ pkgconf \ py3-setuptools python_fuse_version="1.0.9" python_fuse_tarball=$(download_file "https://github.com/libfuse/python-fuse/archive/refs/tags/v$python_fuse_version.tar.gz") python_fuse_tmpdir="$(dirname "$python_fuse_tarball")" execute tar -xzf "$python_fuse_tarball" -C "$python_fuse_tmpdir" execute sh -c "cd '$python_fuse_tmpdir/python-fuse-$python_fuse_version' && python setup.py build" execute_sudo sh -c "cd '$python_fuse_tmpdir/python-fuse-$python_fuse_version' && python setup.py install" # For Alpine we also need to make sure the kernel module is automatically loaded execute_sudo sh -c "echo fuse >> /etc/modules-load.d/fuse.conf" # Check that it was actually installed execute python -c 'import fuse' ;; apt | dnf | yum) install_packages python3-fuse ;; esac } create_buildkite_user() { if ! [ "$ci" = "1" ]; then return fi print "Creating Buildkite user..." user="buildkite-agent" group="$user" home="/var/lib/buildkite-agent" case "$distro" in amzn) install_packages \ shadow-utils \ util-linux ;; esac if [ -z "$(getent passwd "$user")" ]; then case "$distro" in alpine) execute_sudo addgroup \ --system "$group" execute_sudo adduser "$user" \ --system \ --ingroup "$group" \ --shell "$(require sh)" \ --home "$home" \ --disabled-password ;; *) execute_sudo useradd "$user" \ --system \ --shell "$(require sh)" \ --no-create-home \ --home-dir "$home" ;; esac fi if [ -n "$(getent group docker)" ]; then execute_sudo usermod -aG docker "$user" fi buildkite_paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock" for path in $buildkite_paths; do create_directory "$path" done buildkite_files="/var/run/buildkite-agent/buildkite-agent.pid" for file in $buildkite_files; do create_file "$file" done # The following is necessary to configure buildkite to use a stable # checkout directory for ccache to be effective. local opts=$- set -ef # I do not want to use create_file because it creates directories with 777 # permissions and files with 664 permissions. This is dumb, for obvious # reasons. local hook_dir=${home}/hooks mkdir -p -m 755 "${hook_dir}" cat << EOF > "${hook_dir}/environment" #!/bin/sh set -efu export BUILDKITE_BUILD_CHECKOUT_PATH=${home}/build EOF execute_sudo chmod +x "${hook_dir}/environment" execute_sudo chown -R "$user:$group" "$hook_dir" set +ef -"$opts" } install_buildkite() { if ! [ "$ci" = "1" ]; then return fi buildkite_version="3.114.0" case "$arch" in aarch64) buildkite_arch="arm64" ;; x64) buildkite_arch="amd64" ;; esac buildkite_filename="buildkite-agent-$os-$buildkite_arch-$buildkite_version.tar.gz" buildkite_url="https://github.com/buildkite/agent/releases/download/v$buildkite_version/$buildkite_filename" buildkite_tar="$(download_file "$buildkite_url")" buildkite_tmpdir="$(dirname "$buildkite_tar")" execute tar -xzf "$buildkite_tar" -C "$buildkite_tmpdir" move_to_bin "$buildkite_tmpdir/buildkite-agent" } install_chromium() { # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#chrome-doesnt-launch-on-linux # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-the-cloud case "$pm" in apk) install_packages \ chromium \ nss \ freetype \ harfbuzz \ ttf-freefont ;; apt) install_packages \ fonts-liberation \ libatk-bridge2.0-0 \ libatk1.0-0 \ libc6 \ libcairo2 \ libcups2 \ libdbus-1-3 \ libexpat1 \ libfontconfig1 \ libgbm1 \ libgcc1 \ libglib2.0-0 \ libgtk-3-0 \ libnspr4 \ libnss3 \ libpango-1.0-0 \ libpangocairo-1.0-0 \ libstdc++6 \ libx11-6 \ libx11-xcb1 \ libxcb1 \ libxcomposite1 \ libxcursor1 \ libxdamage1 \ libxext6 \ libxfixes3 \ libxi6 \ libxrandr2 \ libxrender1 \ libxss1 \ libxtst6 \ xdg-utils # Fixes issue in newer version of Ubuntu: # Package 'libasound2' has no installation candidate if [ "$(check_package "libasound2t64")" ]; then install_packages libasound2t64 else install_packages libasound2 fi # Install Chrome itself on x64 (no arm64 build exists): with a system # browser present, puppeteer-based tests skip their per-run ~300MB # Chrome for Testing download entirely (see # test/harness.ts getPuppeteerInstallEnv). if [ "$arch" = "x64" ]; then chrome_deb=$(download_file "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb") # Best-effort: execute_sudo aborts the whole script on failure, so the # fallback chain must run inside a single sudo'd shell. execute_sudo sh -c "apt-get install -y '$chrome_deb' || dpkg -i '$chrome_deb' || true" fi ;; dnf | yum) install_packages \ alsa-lib \ atk \ cups-libs \ gtk3 \ ipa-gothic-fonts \ libXcomposite \ libXcursor \ libXdamage \ libXext \ libXi \ libXrandr \ libXScrnSaver \ libXtst \ pango \ xorg-x11-fonts-100dpi \ xorg-x11-fonts-75dpi \ xorg-x11-fonts-cyrillic \ xorg-x11-fonts-misc \ xorg-x11-fonts-Type1 \ xorg-x11-utils ;; pkg) install_packages \ www/chromium \ ;; esac case "$distro" in amzn) install_packages \ mesa-libgbm ;; esac } install_age() { age_version="1.2.1" case "$os" in linux) case "$arch" in x64) age_arch="amd64" age_hash="7df45a6cc87d4da11cc03a539a7470c15b1041ab2b396af088fe9990f7c79d50" ;; aarch64) age_arch="arm64" age_hash="57fd79a7ece5fe501f351b9dd51a82fbee1ea8db65a8839db17f5c080245e99f" ;; *) error "Unsupported platform: $os-$arch" ;; esac ;; *) error "Unsupported platform: $os-$arch" ;; esac age_tarball="$(download_and_verify_file https://github.com/FiloSottile/age/releases/download/v$age_version/age-v$age_version-$os-$age_arch.tar.gz "$age_hash")" age_extract_dir="$(create_tmp_directory)" execute tar -C "$age_extract_dir" -zxf "$age_tarball" age/age move_to_bin "$age_extract_dir/age/age" } configure_core_dumps() { case "$os" in linux) # set up a directory that the test runner will look in after running tests cores_dir="/var/bun-cores-$distro-$release-$arch" sysctl_file="/etc/sysctl.d/local.conf" create_directory "$cores_dir" # ensure core_pattern will point there # %e = executable filename # %p = pid append_file "$sysctl_file" "kernel.core_pattern = $cores_dir/%e-%p.core" # disable apport.service if it exists since it will override the core_pattern if which systemctl >/dev/null; then if systemctl list-unit-files apport.service >/dev/null; then execute_sudo "$systemctl" disable --now apport.service fi fi # load the new configuration (ignore permission errors) execute_sudo sysctl -p "$sysctl_file" # ensure that a regular user will be able to run sysctl if [ -d /sbin ]; then append_to_path /sbin fi # install gdb for backtraces install_packages gdb ;; esac } clean_system() { if ! [ "$ci" = "1" ]; then return fi print "Cleaning system..." tmp_paths="/tmp /var/tmp" for path in $tmp_paths; do execute_sudo rm -rf "$path"/* done case "$pm" in apt) execute_sudo apt-get clean execute_sudo rm -rf /var/lib/apt/lists/* ;; dnf | yum) execute_sudo "$pm" clean all ;; apk) execute_sudo rm -rf /var/cache/apk/* ;; esac if command -v fstrim >/dev/null 2>&1; then if [ "$sudo" = "1" ] || [ -z "$can_sudo" ]; then fstrim -av || true else sudo -n fstrim -av || true fi fi } ensure_no_tmpfs() { if ! [ "$os" = "linux" ]; then return fi if ! ( [ "$distro" = "ubuntu" ] || [ "$distro" = "debian" ] ); then return fi execute_sudo systemctl mask tmp.mount } prefetch_build_deps() { # CI-only: bake a read-only download cache for scripts/build/download.ts # (BUN_BUILD_PREFETCH_DIR). Everything is content-addressed by URL/identity, # so a dep version bump in scripts/build/deps/ just misses the cache for that # one dep — no image rebuild needed. if ! [ "$ci" = "1" ]; then return fi prefetch_dir="/opt/bun-prefetch" bun_path="$(require bun)" git_path="$(require git)" # Only bootstrap.sh is uploaded to the bake VM, so the repo (and the # prefetch script + scripts/build/deps/*.ts version pins) has to be cloned. # BUN_BOOTSTRAP_REPO_REF lets the image-build orchestrator pin to the # commit it was triggered from; default to main. repo_ref="${BUN_BOOTSTRAP_REPO_REF:-main}" clone_dir="$(create_tmp_directory)" # Best-effort: a fork-PR branch that doesn't exist on the upstream remote, # a deleted branch, or a transient network blip shouldn't abort the whole # image bake — the build just falls through to the network with no warm # cache. Same for a ref that predates the prefetch script. if ! "$git_path" clone --depth=1 --branch "$repo_ref" \ https://github.com/oven-sh/bun.git "$clone_dir/bun"; then print "warning: clone of $repo_ref failed; skipping warm cache" execute_sudo rm -rf "$clone_dir" return fi if ! [ -f "$clone_dir/bun/scripts/prefetch-deps.ts" ]; then print "prefetch-deps.ts not present at $repo_ref; skipping warm cache" execute_sudo rm -rf "$clone_dir" return fi create_directory "$prefetch_dir" # resolveConfig() walks up from cwd to find package.json — run from inside # the clone. Direct invocation (not `execute`) so a non-zero is observable # here rather than swallowed by the subshell — the parent shell has no # `set -e`, and `error()` inside a subshell can't kill the parent. if ! ( cd "$clone_dir/bun" && "$bun_path" scripts/prefetch-deps.ts "$prefetch_dir" ); then print "warning: prefetch-deps.ts failed; baking without warm cache" execute_sudo rm -rf "$clone_dir" "$prefetch_dir" return fi # Pre-pull test docker images (postgres, mysql, redis, minio, …) so tests # don't fetch them at runtime. install_docker() only enables the daemon for # next boot on most distros — start it now. Runs as root: install_docker() # added $user to the docker group, but group membership doesn't apply to # the current shell, so a non-root `docker compose` here would get # permission-denied on the socket. Best-effort: a docker hiccup shouldn't # fail the bake. if [ -f "$clone_dir/bun/test/docker/prepare-ci.ts" ] && command -v docker >/dev/null; then systemctl_path="$(which systemctl)" if [ -n "$systemctl_path" ]; then execute_sudo "$systemctl_path" start docker || true fi ( cd "$clone_dir/bun" && execute_sudo "$bun_path" test/docker/prepare-ci.ts ) || \ print "warning: prepare-ci.ts failed; test docker images not pre-pulled" fi # Warm a shared `bun install` download cache so every test shard's # `bun install` (root + test/) hits disk instead of npm. Keyed by # name@version, so a test/package.json bump after the bake just misses for # that one package. Left writable and owned by the buildkite user: bun # install extracts new tarballs into the cache dir itself, so a read-only # cache would fail on the first unseen package rather than fall through. install_cache_dir="/var/cache/bun-install" create_directory "$install_cache_dir" if ( cd "$clone_dir/bun" && \ BUN_INSTALL_CACHE_DIR="$install_cache_dir" "$bun_path" install --ignore-scripts && \ cd test && \ BUN_INSTALL_CACHE_DIR="$install_cache_dir" "$bun_path" install --ignore-scripts ); then # Re-chown after populating: the install ran as the bootstrap user, and # buildkite-agent needs to write new entries alongside the baked ones. grant_to_user "$install_cache_dir" append_file /etc/environment "BUN_INSTALL_CACHE_DIR=$install_cache_dir" append_to_profile "export BUN_INSTALL_CACHE_DIR=\"$install_cache_dir\"" else print "warning: bun install prefetch failed; baking without warm install cache" execute_sudo rm -rf "$install_cache_dir" fi execute_sudo rm -rf "$clone_dir" # Read-only: download.ts only ever copies FROM here, and a writable baked # input is something a misbehaving job could corrupt for later jobs on the # same runner. download.ts also falls back to this path when the env var # isn't set, so the exports below are belt-and-braces for interactive # debugging rather than the load-bearing mechanism. execute_sudo chmod -R a-w "$prefetch_dir" append_file /etc/environment "BUN_BUILD_PREFETCH_DIR=$prefetch_dir" append_to_profile "export BUN_BUILD_PREFETCH_DIR=\"$prefetch_dir\"" } main() { check_features "$@" check_operating_system check_inside_docker check_user check_ulimit check_package_manager create_buildkite_user install_common_software install_build_essentials install_chromium install_fuse_python install_age prefetch_build_deps if [ "${BUN_NO_CORE_DUMP:-0}" != "1" ]; then configure_core_dumps fi clean_system ensure_no_tmpfs } main "$@"