initial commit
This commit is contained in:
i2p
@@ -0,0 +1,278 @@
|
||||
ARG LLVM_VERSION="21"
|
||||
ARG REPORTED_LLVM_VERSION="21.1.8"
|
||||
ARG OLD_BUN_VERSION="1.3.13"
|
||||
ARG BUILDKITE_AGENT_TAGS="queue=linux,os=linux,arch=${TARGETARCH}"
|
||||
|
||||
FROM --platform=$BUILDPLATFORM ubuntu:20.04 as base-arm64
|
||||
FROM --platform=$BUILDPLATFORM ubuntu:20.04 as base-amd64
|
||||
FROM base-$TARGETARCH as base
|
||||
|
||||
ARG LLVM_VERSION
|
||||
ARG OLD_BUN_VERSION
|
||||
ARG TARGETARCH
|
||||
ARG REPORTED_LLVM_VERSION
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
CI=true \
|
||||
DOCKER=true
|
||||
|
||||
RUN echo "Acquire::Queue-Mode \"host\";" > /etc/apt/apt.conf.d/99-apt-queue-mode.conf \
|
||||
&& echo "Acquire::Timeout \"120\";" >> /etc/apt/apt.conf.d/99-apt-timeout.conf \
|
||||
&& echo "Acquire::Retries \"3\";" >> /etc/apt/apt.conf.d/99-apt-retries.conf \
|
||||
&& echo "APT::Install-Recommends \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-recommends.conf \
|
||||
&& echo "APT::Install-Suggests \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-suggests.conf
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget curl git python3 python3-pip ninja-build \
|
||||
software-properties-common apt-transport-https \
|
||||
ca-certificates gnupg lsb-release unzip xz-utils \
|
||||
libxml2-dev ruby ruby-dev bison gawk perl make golang ccache qemu-user-static \
|
||||
nasm \
|
||||
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y gcc-13 g++-13 libgcc-13-dev libstdc++-13-dev \
|
||||
libasan6 libubsan1 libatomic1 libtsan0 liblsan0 \
|
||||
libgfortran5 libc6-dev \
|
||||
&& wget https://apt.llvm.org/llvm.sh \
|
||||
&& chmod +x llvm.sh \
|
||||
&& ./llvm.sh ${LLVM_VERSION} all \
|
||||
&& rm llvm.sh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
RUN --mount=type=tmpfs,target=/tmp \
|
||||
cmake_version="3.30.5" && \
|
||||
if [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-aarch64.sh"; \
|
||||
else \
|
||||
cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh"; \
|
||||
fi && \
|
||||
wget -O /tmp/cmake.sh "$cmake_url" && \
|
||||
sh /tmp/cmake.sh --skip-license --prefix=/usr
|
||||
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \
|
||||
--slave /usr/bin/g++ g++ /usr/bin/g++-13 \
|
||||
--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-13 \
|
||||
--slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \
|
||||
--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13
|
||||
|
||||
RUN echo "ARCH_PATH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64-linux-gnu" || echo "x86_64-linux-gnu")" >> /etc/environment \
|
||||
&& echo "BUN_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x64")" >> /etc/environment
|
||||
|
||||
ENV LD_LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \
|
||||
LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \
|
||||
CPLUS_INCLUDE_PATH=/usr/include/c++/13:/usr/include/${ARCH_PATH}/c++/13 \
|
||||
C_INCLUDE_PATH=/usr/lib/gcc/${ARCH_PATH}/13/include
|
||||
|
||||
RUN if [ "$TARGETARCH" = "arm64" ]; then \
|
||||
export ARCH_PATH="aarch64-linux-gnu"; \
|
||||
else \
|
||||
export ARCH_PATH="x86_64-linux-gnu"; \
|
||||
fi \
|
||||
&& mkdir -p /usr/lib/gcc/${ARCH_PATH}/13 \
|
||||
&& ln -sf /usr/lib/${ARCH_PATH}/libstdc++.so.6 /usr/lib/gcc/${ARCH_PATH}/13/ \
|
||||
&& echo "/usr/lib/gcc/${ARCH_PATH}/13" > /etc/ld.so.conf.d/gcc-13.conf \
|
||||
&& echo "/usr/lib/${ARCH_PATH}" >> /etc/ld.so.conf.d/gcc-13.conf \
|
||||
&& ldconfig
|
||||
|
||||
RUN for f in /usr/lib/llvm-${LLVM_VERSION}/bin/*; do ln -sf "$f" /usr/bin; done \
|
||||
&& ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang \
|
||||
&& ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ \
|
||||
&& ln -sf /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld \
|
||||
&& ln -sf /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb \
|
||||
&& ln -sf /usr/bin/clangd-${LLVM_VERSION} /usr/bin/clangd \
|
||||
&& ln -sf /usr/bin/llvm-ar-${LLVM_VERSION} /usr/bin/llvm-ar \
|
||||
&& ln -sf /usr/bin/ld.lld /usr/bin/ld \
|
||||
&& ln -sf /usr/bin/clang /usr/bin/cc \
|
||||
&& ln -sf /usr/bin/clang++ /usr/bin/c++
|
||||
|
||||
ENV CC="clang" \
|
||||
CXX="clang++" \
|
||||
AR="llvm-ar-${LLVM_VERSION}" \
|
||||
RANLIB="llvm-ranlib-${LLVM_VERSION}" \
|
||||
LD="lld-${LLVM_VERSION}"
|
||||
|
||||
RUN --mount=type=tmpfs,target=/tmp \
|
||||
bash -c '\
|
||||
set -euxo pipefail && \
|
||||
source /etc/environment && \
|
||||
echo "Downloading bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip from https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip" && \
|
||||
curl -fsSL https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip -o /tmp/bun.zip && \
|
||||
unzip /tmp/bun.zip -d /tmp/bun && \
|
||||
mv /tmp/bun/*/bun /usr/bin/bun && \
|
||||
chmod +x /usr/bin/bun'
|
||||
|
||||
# Node — build scripts run under node (see scripts/build.ts). Version must
|
||||
# match scripts/bootstrap.sh nodejs_version_exact() so container and bare-
|
||||
# metal agents use the same runtime. .tar.gz (not .xz) to match bootstrap.sh
|
||||
# and avoid needing xz-utils.
|
||||
ARG NODE_VERSION="24.3.0"
|
||||
RUN ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "x64"; fi) && \
|
||||
curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${ARCH}.tar.gz" \
|
||||
| tar -xz -C /usr/local --strip-components=1 && \
|
||||
node --version
|
||||
|
||||
ENV LLVM_VERSION=${REPORTED_LLVM_VERSION}
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
|
||||
FROM --platform=$BUILDPLATFORM base as buildkite
|
||||
ARG BUILDKITE_AGENT_TAGS
|
||||
|
||||
|
||||
# Install Rust nightly
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
&& export PATH=$HOME/.cargo/bin:$PATH \
|
||||
&& rustup install nightly \
|
||||
&& rustup default nightly \
|
||||
&& rustup target add aarch64-linux-android x86_64-linux-android \
|
||||
&& rustup target add x86_64-unknown-freebsd \
|
||||
&& rustup target add x86_64-pc-windows-msvc aarch64-pc-windows-msvc \
|
||||
&& rustup component add rust-src
|
||||
|
||||
# Android NDK — sysroot/libc++/compiler-rt for --abi=android cross-compile.
|
||||
ARG ANDROID_NDK_VERSION="r27c"
|
||||
RUN curl -fsSL "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip" -o /tmp/ndk.zip \
|
||||
&& unzip -q /tmp/ndk.zip -d /opt \
|
||||
&& mv /opt/android-ndk-${ANDROID_NDK_VERSION} /opt/android-ndk \
|
||||
&& rm /tmp/ndk.zip \
|
||||
# Trim ~1.1GB we don't use (NDK clang/lld, lldb, non-android runtimes) —
|
||||
# we only need sysroot + android compiler-rt. Dramatically shrinks the
|
||||
# docker layer / AMI size.
|
||||
&& rm -rf /opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin \
|
||||
/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/python3 \
|
||||
/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/liblldb.so \
|
||||
/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/*-gnu \
|
||||
/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/*-musl* \
|
||||
/opt/android-ndk/simpleperf /opt/android-ndk/shader-tools /opt/android-ndk/sources \
|
||||
# Symlink NDK compiler-rt builtins + libunwind into host clang's resource
|
||||
# dir — clang's driver hardcodes <resource-dir>/lib/<triple>/libclang_rt.*
|
||||
# with no -L fallback. Done at image-build time (root) since the build
|
||||
# user can't write to /usr.
|
||||
&& RES=$(clang -print-resource-dir) \
|
||||
&& NDK_RT=/opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/$(ls /opt/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/lib/clang/)/lib/linux \
|
||||
&& mkdir -p $RES/lib/linux \
|
||||
&& for A in aarch64 x86_64; do \
|
||||
ln -sf $NDK_RT/libclang_rt.builtins-${A}-android.a $RES/lib/linux/; \
|
||||
mkdir -p $RES/lib/linux/${A}; \
|
||||
ln -sf $NDK_RT/${A}/libunwind.a $RES/lib/linux/${A}/; \
|
||||
DIR=$RES/lib/${A}-unknown-linux-android28; \
|
||||
mkdir -p $DIR; \
|
||||
ln -sf $NDK_RT/libclang_rt.builtins-${A}-android.a $DIR/libclang_rt.builtins.a; \
|
||||
ln -sf $NDK_RT/${A}/libunwind.a $DIR/libunwind.a; \
|
||||
done
|
||||
ENV ANDROID_NDK_ROOT=/opt/android-ndk
|
||||
|
||||
# FreeBSD sysroot — extracted base.txz for --os=freebsd cross-compile.
|
||||
# Only the matching arch is fetched (x64 image gets amd64 sysroot, etc.).
|
||||
ARG FREEBSD_VERSION="14.3"
|
||||
RUN FBSD_ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "amd64"; fi) \
|
||||
&& mkdir -p /opt/freebsd-sysroot \
|
||||
&& curl -fsSL "https://download.freebsd.org/releases/${FBSD_ARCH}/${FREEBSD_VERSION}-RELEASE/base.txz" -o /tmp/base.txz \
|
||||
&& tar -C /opt/freebsd-sysroot --no-same-owner -xJf /tmp/base.txz ./usr/include ./usr/lib ./lib \
|
||||
&& rm /tmp/base.txz
|
||||
ENV FREEBSD_SYSROOT=/opt/freebsd-sysroot
|
||||
|
||||
# Windows sysroot — xwin splat of the MSVC CRT/STL + Windows SDK + ATL (VS
|
||||
# layout) for --os=windows cross-compile; clang-cl/lld-link consume it via
|
||||
# /winsysroot (see scripts/build/config.ts `winsysroot`). Both target arches
|
||||
# in one splat; --include-debug-libs so /MTd debug links work; --include-atl
|
||||
# for <atlstr.h> (rescle.cpp).
|
||||
# --accept-license accepts the Microsoft license terms for the SDK/CRT
|
||||
# components, same as the Windows CI images do when installing VS Build Tools.
|
||||
# If the image predates this layer, configure falls back to fetching the same
|
||||
# splat at build time (scripts/build/winsysroot.ts — keep XWIN_VERSION in sync).
|
||||
# The Include/Lib aliases exist because clang-cl/lld-link compose SDK paths
|
||||
# in title case while the winsysroot-style splat writes lowercase.
|
||||
ARG XWIN_VERSION="0.9.0"
|
||||
RUN XWIN_ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "aarch64"; else echo "x86_64"; fi) \
|
||||
&& curl -fsSL "https://github.com/Jake-Shadle/xwin/releases/download/${XWIN_VERSION}/xwin-${XWIN_VERSION}-${XWIN_ARCH}-unknown-linux-musl.tar.gz" \
|
||||
-o /tmp/xwin.tar.gz \
|
||||
&& tar -xzf /tmp/xwin.tar.gz -C /tmp \
|
||||
&& /tmp/xwin-${XWIN_VERSION}-${XWIN_ARCH}-unknown-linux-musl/xwin --accept-license --arch x86_64,aarch64 --sdk-version 10.0.26100 --crt-version 14.44.17.14 --include-atl --cache-dir /tmp/xwin-cache \
|
||||
splat --use-winsysroot-style --preserve-ms-arch-notation --include-debug-libs --output /opt/winsysroot \
|
||||
> /dev/null \
|
||||
&& ln -s include "/opt/winsysroot/Windows Kits/10/Include" \
|
||||
&& ln -s lib "/opt/winsysroot/Windows Kits/10/Lib" \
|
||||
&& rm -rf /tmp/xwin.tar.gz /tmp/xwin-${XWIN_VERSION}-${XWIN_ARCH}-unknown-linux-musl /tmp/xwin-cache
|
||||
ENV WINDOWS_SYSROOT=/opt/winsysroot
|
||||
|
||||
RUN ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "amd64"; fi) && \
|
||||
echo "Downloading buildkite" && \
|
||||
curl -fsSL "https://github.com/buildkite/agent/releases/download/v3.87.0/buildkite-agent-linux-${ARCH}-3.87.0.tar.gz" -o /tmp/buildkite-agent.tar.gz && \
|
||||
mkdir -p /tmp/buildkite-agent && \
|
||||
tar -xzf /tmp/buildkite-agent.tar.gz -C /tmp/buildkite-agent && \
|
||||
mv /tmp/buildkite-agent/buildkite-agent /usr/bin/buildkite-agent
|
||||
|
||||
RUN mkdir -p /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /etc/buildkite-agent /var/lib/buildkite-agent/cache/bun
|
||||
|
||||
# Warm BUN_BUILD_PREFETCH_DIR (consulted by scripts/build/download.ts before
|
||||
# any network fetch). Content-addressed by URL/identity, so a dep version bump
|
||||
# in scripts/build/deps/ just misses the cache for that one dep — no image
|
||||
# rebuild needed. The clone is only for scripts/prefetch-deps.ts + its
|
||||
# scripts/build/ imports, which aren't in the docker context.
|
||||
ARG BUN_REPO_REF=main
|
||||
RUN set -e; \
|
||||
if git clone --depth=1 --branch ${BUN_REPO_REF} https://github.com/oven-sh/bun.git /tmp/bun-clone \
|
||||
&& [ -f /tmp/bun-clone/scripts/prefetch-deps.ts ]; then \
|
||||
(cd /tmp/bun-clone && bun scripts/prefetch-deps.ts /opt/bun-prefetch); \
|
||||
else \
|
||||
echo "warning: prefetch-deps.ts unavailable at ${BUN_REPO_REF}; skipping warm cache"; \
|
||||
fi; \
|
||||
rm -rf /tmp/bun-clone
|
||||
ENV BUN_BUILD_PREFETCH_DIR=/opt/bun-prefetch
|
||||
|
||||
# The following is necessary to configure buildkite to use a stable
|
||||
# checkout directory for ccache to be effective.
|
||||
RUN mkdir -p -m 755 /var/lib/buildkite-agent/hooks && \
|
||||
cat <<'EOF' > /var/lib/buildkite-agent/hooks/environment
|
||||
#!/bin/sh
|
||||
set -efu
|
||||
|
||||
export BUILDKITE_BUILD_CHECKOUT_PATH=/var/lib/buildkite-agent/build
|
||||
export BUN_BUILD_PREFETCH_DIR=/opt/bun-prefetch
|
||||
EOF
|
||||
|
||||
RUN chmod 744 /var/lib/buildkite-agent/hooks/environment
|
||||
|
||||
COPY ../*/agent.mjs /var/bun/scripts/
|
||||
|
||||
ENV BUN_INSTALL_CACHE=/var/lib/buildkite-agent/cache/bun
|
||||
ENV BUILDKITE_AGENT_TAGS=${BUILDKITE_AGENT_TAGS}
|
||||
|
||||
|
||||
WORKDIR /var/bun/scripts
|
||||
|
||||
ENV PATH=/root/.cargo/bin:$PATH
|
||||
|
||||
|
||||
CMD ["bun", "/var/bun/scripts/agent.mjs", "start"]
|
||||
|
||||
FROM --platform=$BUILDPLATFORM base as bun-build-linux-local
|
||||
|
||||
ARG LLVM_VERSION
|
||||
WORKDIR /workspace/bun
|
||||
|
||||
COPY . /workspace/bun
|
||||
|
||||
|
||||
# Install Rust nightly
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
&& export PATH=$HOME/.cargo/bin:$PATH \
|
||||
&& rustup install nightly \
|
||||
&& rustup default nightly \
|
||||
&& rustup target add aarch64-linux-android x86_64-linux-android \
|
||||
&& rustup target add x86_64-unknown-freebsd \
|
||||
&& rustup component add rust-src
|
||||
|
||||
ENV PATH=/root/.cargo/bin:$PATH
|
||||
|
||||
ENV LLVM_VERSION=${REPORTED_LLVM_VERSION}
|
||||
|
||||
|
||||
RUN --mount=type=tmpfs,target=/workspace/bun/build \
|
||||
ls -la \
|
||||
&& bun run build:release \
|
||||
&& mkdir -p /target \
|
||||
&& cp -r /workspace/bun/build/release/bun /target/bun
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Check if running as root
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "error: must run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check OS compatibility
|
||||
if ! command -v dnf &> /dev/null; then
|
||||
echo "error: this script requires dnf (RHEL/Fedora/CentOS)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure /tmp/agent.mjs, /tmp/Dockerfile are present
|
||||
if [ ! -f /tmp/agent.mjs ] || [ ! -f /tmp/Dockerfile ]; then
|
||||
# Print each missing file
|
||||
if [ ! -f /tmp/agent.mjs ]; then
|
||||
echo "error: /tmp/agent.mjs is missing"
|
||||
fi
|
||||
if [ ! -f /tmp/Dockerfile ]; then
|
||||
echo "error: /tmp/Dockerfile is missing"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install Docker
|
||||
dnf update -y
|
||||
dnf install -y docker
|
||||
|
||||
systemctl enable docker
|
||||
systemctl start docker || {
|
||||
echo "error: failed to start Docker"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Create builder
|
||||
docker buildx create --name builder --driver docker-container --bootstrap --use || {
|
||||
echo "error: failed to create Docker buildx builder"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Set up Docker to start on boot
|
||||
cat << 'EOF' > /etc/systemd/system/buildkite-agent.service
|
||||
[Unit]
|
||||
Description=Buildkite Docker Container
|
||||
After=docker.service network-online.target
|
||||
Requires=docker.service network-online.target
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=0
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
ExecStartPre=-/usr/bin/docker stop buildkite
|
||||
ExecStartPre=-/usr/bin/docker rm buildkite
|
||||
ExecStart=/usr/bin/docker run \
|
||||
--name buildkite \
|
||||
--restart=unless-stopped \
|
||||
--network host \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v /tmp:/tmp \
|
||||
buildkite:latest
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
echo "Building Buildkite image"
|
||||
|
||||
# Clean up any previous build artifacts
|
||||
rm -rf /tmp/fakebun
|
||||
mkdir -p /tmp/fakebun/scripts /tmp/fakebun/.buildkite
|
||||
|
||||
# Copy required files
|
||||
cp /tmp/agent.mjs /tmp/fakebun/scripts/ || {
|
||||
echo "error: failed to copy agent.mjs"
|
||||
exit 1
|
||||
}
|
||||
cp /tmp/Dockerfile /tmp/fakebun/.buildkite/Dockerfile || {
|
||||
echo "error: failed to copy Dockerfile"
|
||||
exit 1
|
||||
}
|
||||
|
||||
cd /tmp/fakebun || {
|
||||
echo "error: failed to change directory"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Build the Buildkite image. BUN_REPO_REF tells the prefetch step which ref's
|
||||
# dep versions to bake — passed through from machine.mjs via BUN_BOOTSTRAP_REPO_REF.
|
||||
docker buildx build \
|
||||
--platform $(uname -m | sed 's/aarch64/linux\/arm64/;s/x86_64/linux\/amd64/') \
|
||||
--tag buildkite:latest \
|
||||
--target buildkite \
|
||||
--build-arg BUN_REPO_REF="${BUN_BOOTSTRAP_REPO_REF:-main}" \
|
||||
-f .buildkite/Dockerfile \
|
||||
--load \
|
||||
. || {
|
||||
echo "error: Docker build failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Pre-pull test docker images (postgres, mysql, redis, minio, …) into the host
|
||||
# daemon so tests don't fetch them at runtime. /var/lib/docker is on the root
|
||||
# volume and survives into the AMI. Best-effort — a missing script or docker
|
||||
# hiccup shouldn't fail the bake.
|
||||
if git clone --depth=1 --branch "${BUN_BOOTSTRAP_REPO_REF:-main}" \
|
||||
https://github.com/oven-sh/bun.git /tmp/bun-test-docker; then
|
||||
if [ -f /tmp/bun-test-docker/test/docker/prepare-ci.ts ]; then
|
||||
(cd /tmp/bun-test-docker && bun test/docker/prepare-ci.ts) || \
|
||||
echo "warning: prepare-ci.ts failed; test docker images not pre-pulled"
|
||||
fi
|
||||
rm -rf /tmp/bun-test-docker
|
||||
fi
|
||||
|
||||
# Create container to ensure image is cached in AMI
|
||||
docker container create \
|
||||
--name buildkite \
|
||||
--restart=unless-stopped \
|
||||
buildkite:latest || {
|
||||
echo "error: failed to create buildkite container"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Reload systemd to pick up new service
|
||||
systemctl daemon-reload
|
||||
|
||||
# Enable the service, but don't start it yet
|
||||
systemctl enable buildkite-agent || {
|
||||
echo "error: failed to enable buildkite-agent service"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "Bootstrap complete"
|
||||
echo "To start the Buildkite agent, run: "
|
||||
echo " systemctl start buildkite-agent"
|
||||
@@ -0,0 +1,16 @@
|
||||
# Uploads the latest CI workflow to Buildkite.
|
||||
# https://buildkite.com/docs/pipelines/defining-steps
|
||||
#
|
||||
# Changes to this file must be manually edited here:
|
||||
# https://buildkite.com/bun/bun/settings/steps
|
||||
steps:
|
||||
- if: "build.pull_request.repository.fork"
|
||||
block: ":eyes:"
|
||||
prompt: "Did you review the PR?"
|
||||
blocked_state: "running"
|
||||
|
||||
- label: ":pipeline:"
|
||||
agents:
|
||||
queue: "build-darwin"
|
||||
command:
|
||||
- "node .buildkite/ci.mjs"
|
||||
Executable
+1724
File diff suppressed because it is too large
Load Diff
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fallback Buildkite annotation for infra failures.
|
||||
#
|
||||
# `scripts/runner.node.mjs` (test steps) and `scripts/build/ci.ts` (build steps)
|
||||
# post their own failure annotations, then set build meta-data key
|
||||
# `reported-$BUILDKITE_JOB_ID` via markBuildkiteStepReported() just before
|
||||
# exiting. Anything that kills the step before that marker is written (agent
|
||||
# command-hook failures such as a tart guest that never boots, artifact
|
||||
# download failures, `node` missing, the runner/build script itself crashing)
|
||||
# leaves the job red with nothing in the build's annotation list, so the
|
||||
# failure is only discoverable by opening the raw log. This repository pre-exit
|
||||
# hook posts a generic annotation for any such job so it shows up alongside
|
||||
# test/build failures.
|
||||
#
|
||||
# The marker is build meta-data, which is server-side and so remains visible
|
||||
# here even when the reporter ran inside an ephemeral VM (the darwin tart
|
||||
# agents forward the Job API socket into the guest). Repository hooks run on
|
||||
# every posix agent, and on Windows via Git Bash. Never exits non-zero.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
status="${BUILDKITE_COMMAND_EXIT_STATUS:-0}"
|
||||
[ "$status" = "0" ] && exit 0
|
||||
[ -n "${BUILDKITE_JOB_ID:-}" ] || exit 0
|
||||
|
||||
# Skip canceled/timed-out jobs: on cancel the agent SIGTERMs then SIGKILLs the
|
||||
# command, so the reporter had no chance to set the marker, and a canceled
|
||||
# build annotating every running job is noise, not signal. The agent's
|
||||
# Executor.Cancel() sets BUILDKITE_JOB_CANCELLED=true in the shell env
|
||||
# (buildkite/agent@3ab8ab31, v3.94.0+), and additionally
|
||||
# BUILDKITE_JOB_TIMED_OUT=true when the cancel was a job-level timeout.
|
||||
if [ "${BUILDKITE_JOB_CANCELLED:-}" = "true" ]; then exit 0; fi
|
||||
# TODO(ci): the exit-code fallback below is only needed while agents older than
|
||||
# v3.94.0 remain in the fleet (as of 2026-07: the linux queue=ci image is still
|
||||
# on v3.87.0 and several bare-metal darwin boxes are on v3.87.0/v3.94.0;
|
||||
# scripts/bootstrap.sh already pins 3.114.0, those images just need rebaking).
|
||||
# -1 is the posix agent-killed code, 3221225786 is Windows STATUS_CONTROL_C_EXIT,
|
||||
# both observed on cancel in build 76127. Delete this `case` once every agent is
|
||||
# v3.94.0+.
|
||||
case "$status" in -1|3221225786) exit 0 ;; esac
|
||||
|
||||
if buildkite-agent meta-data exists "reported-${BUILDKITE_JOB_ID}" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
label="${BUILDKITE_LABEL:-${BUILDKITE_STEP_KEY:-job}}"
|
||||
job_url="${BUILDKITE_BUILD_URL:-}#${BUILDKITE_JOB_ID}"
|
||||
|
||||
# darwin tart agents redirect `tart run` to /tmp/{bk,sc}-<job-id>.log on the
|
||||
# host. A guest that booted and was cleanly stopped writes only
|
||||
# `Stopping VM...`; anything else is tart reporting why it exited.
|
||||
detail=""
|
||||
for log in "/tmp/bk-${BUILDKITE_JOB_ID}.log" "/tmp/sc-${BUILDKITE_JOB_ID}.log"; do
|
||||
[ -s "$log" ] || continue
|
||||
# Cap per-log output so a pathological tart dump cannot push the shared
|
||||
# `--append`ed annotation past Buildkite's 1 MiB body limit.
|
||||
body=$(grep -v '^Stopping VM\.\.\.' "$log" 2>/dev/null | head -c 2048 || true)
|
||||
[ -n "$body" ] && detail+="${detail:+$'\n'}${log##*/}: ${body}"
|
||||
done
|
||||
[ -n "$detail" ] || detail="see the job log for output"
|
||||
|
||||
# Match escapeCodeBlock() in scripts/runner.node.mjs: the body renders inside a
|
||||
# ```terminal fence, so only backticks need escaping.
|
||||
preview=$(printf '%s' "$detail" | sed 's/`/\\`/g')
|
||||
|
||||
printf '<details><summary><a><code>step failed outside runner</code></a> - exit %s on <a href="%s">%s</a></summary>\n\n```terminal\n%s\n```\n\n</details>\n\n' \
|
||||
"$status" "$job_url" "$label" "$preview" \
|
||||
| buildkite-agent annotate --append --style error --context step-failed-outside-runner --priority 5 2>&1 \
|
||||
|| echo "pre-exit: buildkite-agent annotate failed (non-fatal)"
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,288 @@
|
||||
# Batch Windows code signing for all bun-windows-*.zip Buildkite artifacts.
|
||||
#
|
||||
# This runs as a dedicated pipeline step on a Windows x64 agent after all
|
||||
# Windows build-bun steps complete. Signing is done here instead of inline
|
||||
# during each build because DigiCert smctl is x64-only and silently fails
|
||||
# under ARM64 emulation.
|
||||
#
|
||||
# Each zip is downloaded, its exe signed in place, and the zip is re-packed
|
||||
# with the same name so downstream steps (release, tests) see signed binaries.
|
||||
|
||||
param(
|
||||
# Comma-separated list. powershell.exe -File passes everything as
|
||||
# literal strings, so [string[]] with "a,b,c" becomes a 1-element array.
|
||||
[Parameter(Mandatory=$true)]
|
||||
[string]$Artifacts,
|
||||
|
||||
# Comma-separated, same length as Artifacts, mapping each zip to its source step.
|
||||
[Parameter(Mandatory=$true)]
|
||||
[string]$BuildSteps
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
$ArtifactList = $Artifacts -split ","
|
||||
$BuildStepList = $BuildSteps -split ","
|
||||
|
||||
# smctl shells out to signtool.exe which is only in PATH when the VS dev
|
||||
# environment is loaded. Dot-source the existing helper to set it up.
|
||||
. $PSScriptRoot\..\..\scripts\vs-shell.ps1
|
||||
|
||||
function Log-Info {
|
||||
param([string]$Message)
|
||||
Write-Host "[INFO] $Message" -ForegroundColor Cyan
|
||||
}
|
||||
|
||||
function Log-Success {
|
||||
param([string]$Message)
|
||||
Write-Host "[SUCCESS] $Message" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Log-Error {
|
||||
param([string]$Message)
|
||||
Write-Host "[ERROR] $Message" -ForegroundColor Red
|
||||
}
|
||||
|
||||
function Log-Debug {
|
||||
param([string]$Message)
|
||||
if ($env:DEBUG -eq "true" -or $env:DEBUG -eq "1") {
|
||||
Write-Host "[DEBUG] $Message" -ForegroundColor Gray
|
||||
}
|
||||
}
|
||||
|
||||
function Get-BuildkiteSecret {
|
||||
param([string]$Name)
|
||||
$value = & buildkite-agent secret get $Name 2>&1
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrEmpty($value)) {
|
||||
throw "Failed to fetch Buildkite secret: $Name"
|
||||
}
|
||||
return $value
|
||||
}
|
||||
|
||||
function Ensure-Secrets {
|
||||
Log-Info "Fetching signing secrets from Buildkite..."
|
||||
$env:SM_API_KEY = Get-BuildkiteSecret "SM_API_KEY"
|
||||
$env:SM_CLIENT_CERT_PASSWORD = Get-BuildkiteSecret "SM_CLIENT_CERT_PASSWORD"
|
||||
$env:SM_CLIENT_CERT_FILE = Get-BuildkiteSecret "SM_CLIENT_CERT_FILE"
|
||||
$env:SM_KEYPAIR_ALIAS = Get-BuildkiteSecret "SM_KEYPAIR_ALIAS"
|
||||
$env:SM_HOST = Get-BuildkiteSecret "SM_HOST"
|
||||
Log-Success "All signing secrets fetched"
|
||||
}
|
||||
|
||||
function Setup-Certificate {
|
||||
Log-Info "Decoding client certificate..."
|
||||
try {
|
||||
$tempCertPath = Join-Path $env:TEMP "digicert_cert_$(Get-Random).p12"
|
||||
$certBytes = [System.Convert]::FromBase64String($env:SM_CLIENT_CERT_FILE)
|
||||
[System.IO.File]::WriteAllBytes($tempCertPath, $certBytes)
|
||||
$fileSize = (Get-Item $tempCertPath).Length
|
||||
if ($fileSize -lt 100) {
|
||||
throw "Decoded certificate too small: $fileSize bytes"
|
||||
}
|
||||
$env:SM_CLIENT_CERT_FILE = $tempCertPath
|
||||
$script:TempCertPath = $tempCertPath
|
||||
Log-Success "Certificate decoded ($fileSize bytes)"
|
||||
} catch {
|
||||
if (Test-Path $env:SM_CLIENT_CERT_FILE) {
|
||||
Log-Info "Using certificate file path directly: $env:SM_CLIENT_CERT_FILE"
|
||||
} else {
|
||||
throw "SM_CLIENT_CERT_FILE is neither valid base64 nor an existing file"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Install-KeyLocker {
|
||||
Log-Info "Setting up DigiCert KeyLocker tools..."
|
||||
$installDir = "C:\BuildTools\DigiCert"
|
||||
$smctlPath = Join-Path $installDir "smctl.exe"
|
||||
|
||||
if (Test-Path $smctlPath) {
|
||||
Log-Success "smctl already installed at $smctlPath"
|
||||
$env:PATH = "$installDir;$env:PATH"
|
||||
return $smctlPath
|
||||
}
|
||||
|
||||
if (!(Test-Path $installDir)) {
|
||||
New-Item -ItemType Directory -Path $installDir -Force | Out-Null
|
||||
}
|
||||
|
||||
# smctl is x64-only; this script must run on an x64 agent
|
||||
$msiUrl = "https://bun-ci-assets.bun.sh/Keylockertools-windows-x64.msi"
|
||||
$msiPath = Join-Path $env:TEMP "Keylockertools-windows-x64.msi"
|
||||
|
||||
Log-Info "Downloading KeyLocker MSI from $msiUrl"
|
||||
if (Test-Path $msiPath) { Remove-Item $msiPath -Force }
|
||||
(New-Object System.Net.WebClient).DownloadFile($msiUrl, $msiPath)
|
||||
if (!(Test-Path $msiPath)) { throw "MSI download failed" }
|
||||
|
||||
Log-Info "Installing KeyLocker MSI..."
|
||||
$proc = Start-Process -FilePath "msiexec.exe" -Wait -PassThru -NoNewWindow -ArgumentList @(
|
||||
"/i", "`"$msiPath`"",
|
||||
"/quiet", "/norestart",
|
||||
"TARGETDIR=`"$installDir`"",
|
||||
"INSTALLDIR=`"$installDir`"",
|
||||
"ACCEPT_EULA=1",
|
||||
"ADDLOCAL=ALL"
|
||||
)
|
||||
if ($proc.ExitCode -ne 0) {
|
||||
throw "MSI install failed with exit code $($proc.ExitCode)"
|
||||
}
|
||||
|
||||
if (!(Test-Path $smctlPath)) {
|
||||
$found = Get-ChildItem -Path $installDir -Filter "smctl.exe" -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if ($found) {
|
||||
$smctlPath = $found.FullName
|
||||
$installDir = $found.DirectoryName
|
||||
} else {
|
||||
throw "smctl.exe not found after install"
|
||||
}
|
||||
}
|
||||
|
||||
$env:PATH = "$installDir;$env:PATH"
|
||||
Log-Success "smctl installed at $smctlPath"
|
||||
return $smctlPath
|
||||
}
|
||||
|
||||
function Configure-KeyLocker {
|
||||
param([string]$Smctl)
|
||||
Log-Info "Configuring KeyLocker..."
|
||||
|
||||
$version = & $Smctl --version 2>&1
|
||||
Log-Debug "smctl version: $version"
|
||||
|
||||
$saveOut = & $Smctl credentials save $env:SM_API_KEY $env:SM_CLIENT_CERT_PASSWORD 2>&1 | Out-String
|
||||
Log-Debug "credentials save: $saveOut"
|
||||
|
||||
$healthOut = & $Smctl healthcheck 2>&1 | Out-String
|
||||
Log-Debug "healthcheck: $healthOut"
|
||||
if ($healthOut -notlike "*Healthy*" -and $healthOut -notlike "*SUCCESS*" -and $LASTEXITCODE -ne 0) {
|
||||
Log-Error "healthcheck output: $healthOut"
|
||||
# Don't throw — healthcheck is sometimes flaky but signing still works
|
||||
}
|
||||
|
||||
$syncOut = & $Smctl windows certsync 2>&1 | Out-String
|
||||
Log-Debug "certsync: $syncOut"
|
||||
|
||||
Log-Success "KeyLocker configured"
|
||||
}
|
||||
|
||||
function Download-Artifact {
|
||||
param([string]$Name, [string]$StepKey)
|
||||
|
||||
Log-Info "Downloading $Name from step $StepKey"
|
||||
& buildkite-agent artifact download $Name . --step $StepKey
|
||||
if ($LASTEXITCODE -ne 0 -or !(Test-Path $Name)) {
|
||||
throw "Failed to download artifact: $Name"
|
||||
}
|
||||
Log-Success "Downloaded $Name ($((Get-Item $Name).Length) bytes)"
|
||||
}
|
||||
|
||||
function Sign-Exe {
|
||||
param([string]$ExePath, [string]$Smctl)
|
||||
|
||||
$fileName = Split-Path $ExePath -Leaf
|
||||
Log-Info "Signing $fileName ($((Get-Item $ExePath).Length) bytes)..."
|
||||
|
||||
$existing = Get-AuthenticodeSignature $ExePath
|
||||
if ($existing.Status -eq "Valid") {
|
||||
Log-Info "$fileName already signed by $($existing.SignerCertificate.Subject), skipping"
|
||||
return
|
||||
}
|
||||
|
||||
$out = & $Smctl sign --keypair-alias $env:SM_KEYPAIR_ALIAS --input $ExePath --verbose 2>&1 | Out-String
|
||||
Log-Info "smctl output: $out"
|
||||
# smctl exits 0 even on failure — must also check output text
|
||||
if ($LASTEXITCODE -ne 0 -or $out -like "*FAILED*" -or $out -like "*error*") {
|
||||
throw "Signing failed for $fileName (exit $LASTEXITCODE): $out"
|
||||
}
|
||||
|
||||
$sig = Get-AuthenticodeSignature $ExePath
|
||||
if ($sig.Status -ne "Valid") {
|
||||
throw "$fileName signature verification failed: $($sig.Status) - $($sig.StatusMessage)"
|
||||
}
|
||||
Log-Success "$fileName signed by $($sig.SignerCertificate.Subject)"
|
||||
}
|
||||
|
||||
function Sign-Artifact {
|
||||
param([string]$ZipName, [string]$Smctl)
|
||||
|
||||
Write-Host "================================================" -ForegroundColor Cyan
|
||||
Write-Host " Signing $ZipName" -ForegroundColor Cyan
|
||||
Write-Host "================================================" -ForegroundColor Cyan
|
||||
|
||||
$extractDir = [System.IO.Path]::GetFileNameWithoutExtension($ZipName)
|
||||
|
||||
if (Test-Path $extractDir) { Remove-Item $extractDir -Recurse -Force }
|
||||
|
||||
Log-Info "Extracting $ZipName"
|
||||
Expand-Archive -Path $ZipName -DestinationPath . -Force
|
||||
if (!(Test-Path $extractDir)) {
|
||||
throw "Expected directory $extractDir not found after extraction"
|
||||
}
|
||||
|
||||
$exes = Get-ChildItem -Path $extractDir -Filter "*.exe"
|
||||
if ($exes.Count -eq 0) {
|
||||
throw "No .exe files found in $extractDir"
|
||||
}
|
||||
|
||||
foreach ($exe in $exes) {
|
||||
Sign-Exe -ExePath $exe.FullName -Smctl $Smctl
|
||||
}
|
||||
|
||||
Log-Info "Re-packing $ZipName"
|
||||
Remove-Item $ZipName -Force
|
||||
# Use the same zip command as the build-bun step (scripts/build/ci.ts makeZip)
|
||||
# so the signed archive's entry layout matches the original: forward-slash
|
||||
# paths and a directory entry. Compress-Archive writes backslash separators,
|
||||
# which violates the ZIP spec and triggers warnings in non-Windows unzip.
|
||||
& cmake -E tar cfv $ZipName --format=zip $extractDir
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "cmake -E tar failed for $ZipName"
|
||||
}
|
||||
Remove-Item $extractDir -Recurse -Force
|
||||
|
||||
Log-Info "Uploading signed $ZipName"
|
||||
& buildkite-agent artifact upload $ZipName
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to upload $ZipName"
|
||||
}
|
||||
|
||||
Log-Success "$ZipName signed and uploaded"
|
||||
}
|
||||
|
||||
# Main
|
||||
try {
|
||||
Write-Host "================================================" -ForegroundColor Cyan
|
||||
Write-Host " Windows Artifact Code Signing" -ForegroundColor Cyan
|
||||
Write-Host "================================================" -ForegroundColor Cyan
|
||||
|
||||
if ($ArtifactList.Count -ne $BuildStepList.Count) {
|
||||
throw "Artifact count ($($ArtifactList.Count)) must match BuildStep count ($($BuildStepList.Count))"
|
||||
}
|
||||
Log-Info "Will sign $($ArtifactList.Count) artifacts: $($ArtifactList -join ', ')"
|
||||
|
||||
Ensure-Secrets
|
||||
Setup-Certificate
|
||||
$smctl = Install-KeyLocker
|
||||
Configure-KeyLocker -Smctl $smctl
|
||||
|
||||
for ($i = 0; $i -lt $ArtifactList.Count; $i++) {
|
||||
Download-Artifact -Name $ArtifactList[$i] -StepKey $BuildStepList[$i]
|
||||
Sign-Artifact -ZipName $ArtifactList[$i] -Smctl $smctl
|
||||
}
|
||||
|
||||
Write-Host "================================================" -ForegroundColor Green
|
||||
Write-Host " All artifacts signed successfully" -ForegroundColor Green
|
||||
Write-Host "================================================" -ForegroundColor Green
|
||||
exit 0
|
||||
|
||||
} catch {
|
||||
Log-Error "Signing failed: $_"
|
||||
exit 1
|
||||
|
||||
} finally {
|
||||
if ($script:TempCertPath -and (Test-Path $script:TempCertPath)) {
|
||||
Remove-Item $script:TempCertPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
Executable
+409
@@ -0,0 +1,409 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
function assert_main() {
|
||||
if [ -z "$BUILDKITE_REPO" ]; then
|
||||
echo "error: Cannot find repository for this build"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$BUILDKITE_COMMIT" ]; then
|
||||
echo "error: Cannot find commit for this build"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$BUILDKITE_PULL_REQUEST_REPO" ] && [ "$BUILDKITE_REPO" != "$BUILDKITE_PULL_REQUEST_REPO" ]; then
|
||||
echo "error: Cannot upload release from a fork"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
|
||||
echo "error: Cannot upload release from a pull request"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$BUILDKITE_BRANCH" != "main" ]; then
|
||||
echo "error: Cannot upload release from a branch other than main"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function assert_buildkite_agent() {
|
||||
if ! command -v "buildkite-agent" &> /dev/null; then
|
||||
echo "error: Cannot find buildkite-agent, please install it:"
|
||||
echo "https://buildkite.com/docs/agent/v3/install"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function assert_github() {
|
||||
assert_command "gh" "gh" "https://github.com/cli/cli#installation"
|
||||
assert_buildkite_secret "GITHUB_TOKEN"
|
||||
# gh expects the token in $GH_TOKEN
|
||||
export GH_TOKEN="$GITHUB_TOKEN"
|
||||
}
|
||||
|
||||
function assert_aws() {
|
||||
assert_command "aws" "awscli" "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
|
||||
for secret in "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "AWS_ENDPOINT"; do
|
||||
assert_buildkite_secret "$secret"
|
||||
done
|
||||
assert_buildkite_secret "AWS_BUCKET" --skip-redaction
|
||||
}
|
||||
|
||||
function assert_sentry() {
|
||||
assert_command "sentry-cli" "getsentry/tools/sentry-cli" "https://docs.sentry.io/cli/installation/"
|
||||
for secret in "SENTRY_AUTH_TOKEN" "SENTRY_ORG" "SENTRY_PROJECT"; do
|
||||
assert_buildkite_secret "$secret"
|
||||
done
|
||||
}
|
||||
|
||||
function run_command() {
|
||||
set -x
|
||||
"$@"
|
||||
{ local status=$?; set +x; } 2>/dev/null
|
||||
return "$status"
|
||||
}
|
||||
|
||||
# Zips are read with unzip and written with cmake. Not one tool for both:
|
||||
# `cmake -E tar xf` streams, so it exits 0 on a truncated archive and leaves a
|
||||
# corrupt file behind where unzip exits 9, and `zip` is not on the agent image
|
||||
# (which has no root to install it). cmake is what wrote these zips in the
|
||||
# first place — scripts/build/ci.ts makeZip.
|
||||
function assert_archive_tools() {
|
||||
for tool in "unzip" "cmake"; do
|
||||
if ! command -v "$tool" &> /dev/null; then
|
||||
echo "error: Cannot find $tool"
|
||||
echo ""
|
||||
echo "hint: the agent image is supposed to have it; see scripts/bootstrap.sh"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Tools this script installs go to a writable directory on PATH instead of
|
||||
# /usr/local/bin, which needs root on most agents.
|
||||
function ensure_tools_bin() {
|
||||
if [ -n "$TOOLS_BIN" ]; then
|
||||
return
|
||||
fi
|
||||
TOOLS_DIR="${HOME:-}/.cache/bun-release-tools"
|
||||
if [ -z "$HOME" ] || ! mkdir -p "$TOOLS_DIR/bin" 2> /dev/null; then
|
||||
TOOLS_DIR="$(mktemp -d)"
|
||||
mkdir -p "$TOOLS_DIR/bin"
|
||||
fi
|
||||
TOOLS_BIN="$TOOLS_DIR/bin"
|
||||
export PATH="$TOOLS_BIN:$PATH"
|
||||
}
|
||||
|
||||
function install_gh_linux() {
|
||||
local arch
|
||||
case "$(uname -m)" in
|
||||
x86_64 | amd64) arch="amd64" ;;
|
||||
aarch64 | arm64) arch="arm64" ;;
|
||||
*) echo "error: Unsupported architecture: $(uname -m)"; exit 1 ;;
|
||||
esac
|
||||
# Resolve the version from the releases/latest redirect, not the REST API: the API is rate
|
||||
# limited to 60 req/hour per IP (GITHUB_TOKEN is not exported yet), and piping curl into a
|
||||
# short-circuiting reader such as `grep -m1` makes curl exit 23 (EPIPE) under pipefail.
|
||||
local url version
|
||||
url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "https://github.com/cli/cli/releases/latest")"
|
||||
version="${url##*/tag/v}"
|
||||
if [ -z "$version" ] || [ "$version" == "$url" ]; then
|
||||
echo "error: Cannot determine latest gh release version from: $url"
|
||||
exit 1
|
||||
fi
|
||||
local dir
|
||||
dir="$(mktemp -d)"
|
||||
run_command curl -fsSL "https://github.com/cli/cli/releases/download/v${version}/gh_${version}_linux_${arch}.tar.gz" -o "$dir/gh.tar.gz"
|
||||
run_command tar -xzf "$dir/gh.tar.gz" -C "$dir" --strip-components=1
|
||||
ensure_tools_bin
|
||||
run_command install -m 0755 "$dir/bin/gh" "$TOOLS_BIN/gh"
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
function install_aws_linux() {
|
||||
local dir
|
||||
dir="$(mktemp -d)"
|
||||
run_command curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "$dir/awscliv2.zip"
|
||||
run_command unzip -q "$dir/awscliv2.zip" -d "$dir"
|
||||
ensure_tools_bin
|
||||
run_command "$dir/aws/install" --update -i "$TOOLS_DIR/aws-cli" -b "$TOOLS_BIN"
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
function install_sentry_cli_linux() {
|
||||
# The installer drops a single static binary into INSTALL_DIR.
|
||||
ensure_tools_bin
|
||||
run_command bash -c "curl -fsSL https://sentry.io/get-cli/ | INSTALL_DIR='$TOOLS_BIN' sh"
|
||||
}
|
||||
|
||||
function assert_command() {
|
||||
local command="$1"
|
||||
local package="$2"
|
||||
local help_url="$3"
|
||||
if command -v "$command" &> /dev/null; then
|
||||
return
|
||||
fi
|
||||
echo "warning: $command is not installed, installing..."
|
||||
if command -v brew &> /dev/null; then
|
||||
HOMEBREW_NO_AUTO_UPDATE=1 run_command brew install "$package"
|
||||
elif [ "$(uname -s)" == "Linux" ]; then
|
||||
case "$command" in
|
||||
gh) install_gh_linux ;;
|
||||
aws) install_aws_linux ;;
|
||||
sentry-cli) install_sentry_cli_linux ;;
|
||||
*) echo "error: Don't know how to install $command on Linux"; exit 1 ;;
|
||||
esac
|
||||
else
|
||||
echo "error: Cannot install $command, please install it"
|
||||
if [ -n "$help_url" ]; then
|
||||
echo ""
|
||||
echo "hint: See $help_url for help"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v "$command" &> /dev/null; then
|
||||
echo "error: Failed to install $command"
|
||||
if [ -n "$help_url" ]; then
|
||||
echo ""
|
||||
echo "hint: See $help_url for help"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function assert_buildkite_secret() {
|
||||
local key="$1"
|
||||
local value=$(buildkite-agent secret get "$key" ${@:2})
|
||||
if [ -z "$value" ]; then
|
||||
echo "error: Cannot find $key secret"
|
||||
echo ""
|
||||
echo "hint: Create a secret named $key with a value:"
|
||||
echo "https://buildkite.com/docs/pipelines/buildkite-secrets"
|
||||
exit 1
|
||||
fi
|
||||
export "$key"="$value"
|
||||
}
|
||||
|
||||
function release_tag() {
|
||||
local version="$1"
|
||||
if [ "$version" == "canary" ]; then
|
||||
echo "canary"
|
||||
else
|
||||
echo "bun-v$version"
|
||||
fi
|
||||
}
|
||||
|
||||
function create_sentry_release() {
|
||||
local version="$1"
|
||||
local release="$version"
|
||||
if [ "$version" == "canary" ]; then
|
||||
release="$BUILDKITE_COMMIT-canary"
|
||||
fi
|
||||
run_command sentry-cli releases new "$release" --finalize
|
||||
run_command sentry-cli releases set-commits "$release" --auto --ignore-missing
|
||||
if [ "$version" == "canary" ]; then
|
||||
run_command sentry-cli deploys new --env="canary" --release="$release"
|
||||
fi
|
||||
}
|
||||
|
||||
function download_buildkite_artifact() {
|
||||
local name="$1"
|
||||
local dir="$2"
|
||||
if [ -z "$dir" ]; then
|
||||
dir="."
|
||||
fi
|
||||
# When signing ran, Windows zips exist in two steps with the same name
|
||||
# (build-bun unsigned, windows-sign signed). Pin to the sign step to
|
||||
# guarantee we get the signed one.
|
||||
local step_args=()
|
||||
if [[ -n "$WINDOWS_ARTIFACT_STEP" && "$name" == bun-windows-* ]]; then
|
||||
step_args=(--step "$WINDOWS_ARTIFACT_STEP")
|
||||
fi
|
||||
run_command buildkite-agent artifact download "$name" "$dir" "${step_args[@]}"
|
||||
if [ ! -f "$dir/$name" ]; then
|
||||
echo "error: Cannot find Buildkite artifact: $name"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function upload_github_assets() {
|
||||
local tag="$(release_tag "$1")"
|
||||
run_command gh release upload "$tag" "${@:2}" --clobber --repo "$BUILDKITE_REPO"
|
||||
}
|
||||
|
||||
function update_github_release() {
|
||||
local version="$1"
|
||||
local tag="$(release_tag "$version")"
|
||||
if [ "$tag" == "canary" ]; then
|
||||
run_command gh release edit "$tag" --repo "$BUILDKITE_REPO" \
|
||||
--notes "This release of Bun corresponds to the commit: $BUILDKITE_COMMIT"
|
||||
fi
|
||||
}
|
||||
|
||||
# S3 is a mirror; `bun upgrade` and install.sh read the GitHub release. A
|
||||
# canary that made it to GitHub but not S3 has shipped, so don't fail it.
|
||||
function upload_s3_files() {
|
||||
local version="$1"
|
||||
local files=("${@:2}")
|
||||
local commit_folder="releases/$BUILDKITE_COMMIT"
|
||||
if [ "$version" == "canary" ]; then
|
||||
commit_folder="$commit_folder-canary"
|
||||
fi
|
||||
local status=0 file
|
||||
for file in "${files[@]}"; do
|
||||
run_command aws --endpoint-url="$AWS_ENDPOINT" s3 cp "$file" "s3://$AWS_BUCKET/$commit_folder/$file" || status=1
|
||||
run_command aws --endpoint-url="$AWS_ENDPOINT" s3 cp "$file" "s3://$AWS_BUCKET/releases/$version/$file" || status=1
|
||||
done
|
||||
if [ "$status" -eq 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$version" == "canary" ]; then
|
||||
echo "warn: Some S3 uploads failed, ignoring since this is a canary release"
|
||||
return 0
|
||||
fi
|
||||
echo "error: Some S3 uploads failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function send_discord_announcement() {
|
||||
local value=$(buildkite-agent secret get "BUN_ANNOUNCE_CANARY_WEBHOOK_URL")
|
||||
if [ -z "$value" ]; then
|
||||
echo "warn: BUN_ANNOUNCE_CANARY_WEBHOOK_URL not set, skipping Discord announcement"
|
||||
return
|
||||
fi
|
||||
|
||||
local version="$1"
|
||||
local commit="$BUILDKITE_COMMIT"
|
||||
local short_sha="${commit:0:7}"
|
||||
local commit_url="https://github.com/oven-sh/bun/commit/$commit"
|
||||
|
||||
if [ "$version" == "canary" ]; then
|
||||
local json_payload=$(cat <<EOF
|
||||
{
|
||||
"embeds": [{
|
||||
"title": "New Bun Canary now available",
|
||||
"description": "A new canary build of Bun has been automatically uploaded ([${short_sha}](${commit_url})). To upgrade, run:\n\n\`\`\`shell\nbun upgrade --canary\n\`\`\`\nCommit: \`${commit}\`",
|
||||
"color": 16023551,
|
||||
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
}]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
curl -H "Content-Type: application/json" \
|
||||
-d "$json_payload" \
|
||||
-sf \
|
||||
"$value" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
function create_release() {
|
||||
assert_main
|
||||
assert_buildkite_agent
|
||||
assert_archive_tools
|
||||
assert_github
|
||||
assert_aws
|
||||
assert_sentry
|
||||
|
||||
local tag="$1" # 'canary' or 'x.y.z'
|
||||
local artifacts=(
|
||||
bun-darwin-aarch64.zip
|
||||
bun-darwin-aarch64-profile.zip
|
||||
bun-darwin-x64.zip
|
||||
bun-darwin-x64-profile.zip
|
||||
bun-linux-aarch64.zip
|
||||
bun-linux-aarch64-profile.zip
|
||||
bun-linux-x64.zip
|
||||
bun-linux-x64-profile.zip
|
||||
bun-linux-aarch64-musl.zip
|
||||
bun-linux-aarch64-musl-profile.zip
|
||||
bun-linux-x64-musl.zip
|
||||
bun-linux-x64-musl-profile.zip
|
||||
bun-linux-aarch64-android.zip
|
||||
bun-linux-aarch64-android-profile.zip
|
||||
bun-linux-x64-android.zip
|
||||
bun-linux-x64-android-profile.zip
|
||||
bun-freebsd-aarch64.zip
|
||||
bun-freebsd-aarch64-profile.zip
|
||||
bun-freebsd-x64.zip
|
||||
bun-freebsd-x64-profile.zip
|
||||
bun-windows-x64.zip
|
||||
bun-windows-x64-profile.zip
|
||||
bun-windows-aarch64.zip
|
||||
bun-windows-aarch64-profile.zip
|
||||
)
|
||||
|
||||
# x64 ships one nehalem binary under the plain name. Re-zip it under the
|
||||
# historical `-baseline` name (inner dir renamed) so older `bun upgrade`
|
||||
# clients that still request `-baseline` extract correctly.
|
||||
function alias_baseline_artifact() {
|
||||
local artifact="$1"
|
||||
case "$artifact" in
|
||||
bun-darwin-x64.zip) echo "bun-darwin-x64-baseline.zip" ;;
|
||||
bun-darwin-x64-profile.zip) echo "bun-darwin-x64-baseline-profile.zip" ;;
|
||||
bun-linux-x64.zip) echo "bun-linux-x64-baseline.zip" ;;
|
||||
bun-linux-x64-profile.zip) echo "bun-linux-x64-baseline-profile.zip" ;;
|
||||
bun-linux-x64-musl.zip) echo "bun-linux-x64-musl-baseline.zip" ;;
|
||||
bun-linux-x64-musl-profile.zip) echo "bun-linux-x64-musl-baseline-profile.zip" ;;
|
||||
bun-windows-x64.zip) echo "bun-windows-x64-baseline.zip" ;;
|
||||
bun-windows-x64-profile.zip) echo "bun-windows-x64-baseline-profile.zip" ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Repack `$src_zip` (inner dir = basename of $src_zip) as `$dst_zip` with the
|
||||
# inner dir renamed to match `$dst_zip`'s basename, which is what install.sh
|
||||
# extracts. Not done in the build step's makeZip, where the staging dir is
|
||||
# already in hand: the Windows zips are re-uploaded by the signing step, so
|
||||
# an alias built there would carry the unsigned binary. Runs in a fresh
|
||||
# mktemp dir so a caller-CWD change can't collide with the extracted names.
|
||||
function rezip_as() {
|
||||
local src_zip="$1" dst_zip="$2"
|
||||
local src_dir="${src_zip%.zip}" dst_dir="${dst_zip%.zip}"
|
||||
local abs_src="$PWD/$src_zip" abs_dst="$PWD/$dst_zip"
|
||||
local work; work="$(mktemp -d)"
|
||||
run_command unzip -q -d "$work" "$abs_src"
|
||||
run_command mv "$work/$src_dir" "$work/$dst_dir"
|
||||
(cd "$work" && run_command cmake -E tar cf "$abs_dst" --format=zip "$dst_dir")
|
||||
run_command rm -rf "$work"
|
||||
}
|
||||
|
||||
# Fetch everything up front so the GitHub release can take all assets in one
|
||||
# `gh release upload`; per-file uploads raced on the same release.
|
||||
local files=() pids=() artifact
|
||||
for artifact in "${artifacts[@]}"; do
|
||||
download_buildkite_artifact "$artifact" & pids+=("$!")
|
||||
files+=("$artifact")
|
||||
done
|
||||
# Per-pid: a bare `wait` returns 0 however the children exited.
|
||||
local pid status=0
|
||||
for pid in "${pids[@]}"; do
|
||||
wait "$pid" || status=1
|
||||
done
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "error: Failed to download one or more Buildkite artifacts"
|
||||
exit 1
|
||||
fi
|
||||
for artifact in "${artifacts[@]}"; do
|
||||
local alias="$(alias_baseline_artifact "$artifact")"
|
||||
if [ -n "$alias" ]; then
|
||||
rezip_as "$artifact" "$alias"
|
||||
files+=("$alias")
|
||||
fi
|
||||
done
|
||||
|
||||
upload_github_assets "$tag" "${files[@]}"
|
||||
update_github_release "$tag"
|
||||
create_sentry_release "$tag"
|
||||
send_discord_announcement "$tag"
|
||||
upload_s3_files "$tag" "${files[@]}"
|
||||
}
|
||||
|
||||
function assert_canary() {
|
||||
if [ -z "$CANARY" ] || [ "$CANARY" == "0" ]; then
|
||||
echo "warn: Skipping release because this is not a canary build"
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
assert_canary
|
||||
create_release "canary"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Regenerates test/expected-durations.json from recent Buildkite runs and
|
||||
# uploads it as a build artifact. Attach a weekly schedule to this pipeline in
|
||||
# the Buildkite UI (it is not wired into ci.mjs so it never runs on PRs).
|
||||
#
|
||||
# The runner uses the checked-in copy for sharding; refresh that copy by
|
||||
# downloading this artifact and committing it when the shard balance drifts.
|
||||
steps:
|
||||
- label: ":stopwatch: update-test-durations"
|
||||
if: build.source == "schedule" || build.source == "ui"
|
||||
agents:
|
||||
queue: build-linux
|
||||
command: |
|
||||
node scripts/update-test-durations.mjs --builds 5
|
||||
node scripts/update-parallel-allowlist.mjs --builds 300
|
||||
buildkite-agent artifact upload test/expected-durations.json
|
||||
buildkite-agent artifact upload test/parallel-allowlist.json
|
||||
env:
|
||||
# The script reads BUILDKITE_API_TOKEN; the agent environment hook
|
||||
# already exports a read-scoped token under this name.
|
||||
BUILDKITE_API_TOKEN: "$BUILDKITE_API_TOKEN"
|
||||
Reference in New Issue
Block a user