Wire runner caches and forge secrets through agenix
This commit is contained in:
parent
afc3e79eb0
commit
ed247b2f5e
20 changed files with 299 additions and 64 deletions
|
|
@ -22,14 +22,17 @@ jobs:
|
|||
matrix:
|
||||
include:
|
||||
- platform: macOS
|
||||
cache-id: macos
|
||||
destination: platform=macOS
|
||||
rust-targets: x86_64-apple-darwin,aarch64-apple-darwin
|
||||
- platform: iOS Simulator
|
||||
cache-id: ios-simulator
|
||||
destination: platform=iOS Simulator,name=iPhone 17 Pro
|
||||
rust-targets: aarch64-apple-ios-sim,x86_64-apple-ios
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUST_BACKTRACE: short
|
||||
RUSTC_WRAPPER: sccache
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://code.forgejo.org/actions/checkout@v4
|
||||
|
|
@ -65,12 +68,29 @@ jobs:
|
|||
echo "DEVELOPER_DIR=$selected" >> "$GITHUB_ENV"
|
||||
DEVELOPER_DIR="$selected" /usr/bin/xcodebuild -version || true
|
||||
|
||||
- name: Prepare Cache Dirs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}"
|
||||
mkdir -p \
|
||||
"${cache_root}/cargo" \
|
||||
"${cache_root}/rustup" \
|
||||
"${cache_root}/sccache" \
|
||||
"${cache_root}/apple/PackageCache" \
|
||||
"${cache_root}/apple/SourcePackages" \
|
||||
"${cache_root}/apple/DerivedData/${{ matrix.cache-id }}"
|
||||
echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}"
|
||||
echo "RUSTUP_HOME=${cache_root}/rustup" >> "${GITHUB_ENV}"
|
||||
echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}"
|
||||
echo "APPLE_PACKAGE_CACHE=${cache_root}/apple/PackageCache" >> "${GITHUB_ENV}"
|
||||
echo "APPLE_SOURCE_PACKAGES=${cache_root}/apple/SourcePackages" >> "${GITHUB_ENV}"
|
||||
echo "APPLE_DERIVED_DATA=${cache_root}/apple/DerivedData/${{ matrix.cache-id }}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Install Rust
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export RUSTUP_HOME="${HOME}/.rustup"
|
||||
export CARGO_HOME="${HOME}/.cargo"
|
||||
|
||||
if ! command -v rustup >/dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.85.0
|
||||
|
|
@ -98,6 +118,9 @@ jobs:
|
|||
if ! command -v protoc >/dev/null 2>&1; then
|
||||
brew install protobuf
|
||||
fi
|
||||
if ! command -v sccache >/dev/null 2>&1; then
|
||||
brew install sccache
|
||||
fi
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
|
|
@ -111,9 +134,9 @@ jobs:
|
|||
-skipPackagePluginValidation \
|
||||
-skipMacroValidation \
|
||||
-onlyUsePackageVersionsFromResolvedFile \
|
||||
-clonedSourcePackagesDirPath SourcePackages \
|
||||
-packageCachePath "$PWD/PackageCache" \
|
||||
-derivedDataPath "$PWD/DerivedData" \
|
||||
-clonedSourcePackagesDirPath "$APPLE_SOURCE_PACKAGES" \
|
||||
-packageCachePath "$APPLE_PACKAGE_CACHE" \
|
||||
-derivedDataPath "$APPLE_DERIVED_DATA" \
|
||||
CODE_SIGNING_ALLOWED=NO \
|
||||
CODE_SIGNING_REQUIRED=NO \
|
||||
CODE_SIGN_IDENTITY="" \
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ jobs:
|
|||
rust:
|
||||
name: Cargo Test
|
||||
runs-on: [self-hosted, linux, x86_64, burrow-forge]
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_CACHE_SIZE: 20G
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://code.forgejo.org/actions/checkout@v4
|
||||
|
|
@ -24,8 +28,21 @@ jobs:
|
|||
token: ${{ github.token }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Prepare Cache Dirs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cache_root="${HOME}/.cache/burrow"
|
||||
mkdir -p "${cache_root}/cargo" "${cache_root}/sccache"
|
||||
echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}"
|
||||
echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
nix develop .#ci -c cargo test --workspace --all-features
|
||||
nix develop .#ci -c bash -lc '
|
||||
sccache --zero-stats >/dev/null 2>&1 || true
|
||||
cargo test --workspace --all-features
|
||||
sccache --show-stats || true
|
||||
'
|
||||
|
|
|
|||
|
|
@ -24,6 +24,14 @@ jobs:
|
|||
token: ${{ github.token }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Prepare Cache Dirs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cache_root="${HOME}/.cache/burrow"
|
||||
mkdir -p "${cache_root}/npm"
|
||||
echo "NPM_CONFIG_CACHE=${cache_root}/npm" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -73,7 +73,13 @@ CARGO_PATH="$(dirname $PROTOC):$CARGO_PATH"
|
|||
|
||||
# Run cargo without the various environment variables set by Xcode.
|
||||
# Those variables can confuse cargo and the build scripts it runs.
|
||||
env -i PATH="$CARGO_PATH" PROTOC="$PROTOC" CARGO_TARGET_DIR="${CONFIGURATION_TEMP_DIR}/target" IPHONEOS_DEPLOYMENT_TARGET="$IPHONEOS_DEPLOYMENT_TARGET" MACOSX_DEPLOYMENT_TARGET="$MACOSX_DEPLOYMENT_TARGET" cargo build "${CARGO_ARGS[@]}"
|
||||
EXTRA_ENV=()
|
||||
for VAR_NAME in HOME CARGO_HOME RUSTUP_HOME RUSTC_WRAPPER SCCACHE_DIR CARGO_INCREMENTAL; do
|
||||
if [[ -n "${!VAR_NAME:-}" ]]; then
|
||||
EXTRA_ENV+=("${VAR_NAME}=${!VAR_NAME}")
|
||||
fi
|
||||
done
|
||||
env -i PATH="$CARGO_PATH" PROTOC="$PROTOC" CARGO_TARGET_DIR="${CONFIGURATION_TEMP_DIR}/target" IPHONEOS_DEPLOYMENT_TARGET="$IPHONEOS_DEPLOYMENT_TARGET" MACOSX_DEPLOYMENT_TARGET="$MACOSX_DEPLOYMENT_TARGET" "${EXTRA_ENV[@]}" cargo build "${CARGO_ARGS[@]}"
|
||||
|
||||
mkdir -p "${BUILT_PRODUCTS_DIR}"
|
||||
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -1,7 +1,9 @@
|
|||
FLAKE ?= .
|
||||
AGENIX ?= nix run ${FLAKE}\#agenix --
|
||||
|
||||
SECRETS := forgejo/nsc-token \
|
||||
SECRETS := forgejo/admin-password \
|
||||
forgejo/agent-ssh-key \
|
||||
forgejo/nsc-token \
|
||||
forgejo/nsc-dispatcher-config \
|
||||
forgejo/nsc-autoscaler-config
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
agenixPkg = agenix.packages.${system}.agenix;
|
||||
commonPackages = with pkgs; [
|
||||
cargo
|
||||
sccache
|
||||
rustc
|
||||
rustfmt
|
||||
clippy
|
||||
|
|
|
|||
|
|
@ -15,19 +15,19 @@ Mail hosting is intentionally not part of this NixOS host in the current plan. B
|
|||
- `keys/agent_at_burrow_net.pub`: automation SSH public key
|
||||
- `../Scripts/hetzner-forge.sh`: Hetzner inventory and replace workflow
|
||||
- `../Scripts/nsc-build-and-upload-image.sh`: temporary Namespace builder -> raw image -> Hetzner snapshot
|
||||
- `../Scripts/bootstrap-forge-intake.sh`: copy the Forgejo bootstrap password and agent SSH key into `/var/lib/burrow/intake/`
|
||||
- `../Scripts/bootstrap-forge-intake.sh`: legacy intake bootstrap helper; current forge runtime secrets should live in `../secrets/forgejo/*.age`
|
||||
- `../Scripts/check-forge-host.sh`: verify Forgejo, Caddy, the local runner, and optional NSC services after boot
|
||||
- `../Scripts/cloudflare-upsert-a-record.sh`: upsert DNS-only Cloudflare `A` records for Burrow host cutovers
|
||||
- `../Scripts/forge-deploy.sh`: remote `nixos-rebuild` entrypoint for the forge host
|
||||
- `../Scripts/provision-forgejo-nsc.sh`: render Burrow Namespace dispatcher/autoscaler bootstrap inputs and ensure the default Forgejo scope exists
|
||||
- `../secrets/forgejo/*.age`: authoritative encrypted Namespace token + dispatcher/autoscaler configs for the forge host
|
||||
- `../secrets/forgejo/*.age`: authoritative encrypted forge admin password, agent SSH key, and Namespace runtime configs for the forge host
|
||||
|
||||
## Intended Flow
|
||||
|
||||
1. Build and upload the raw NixOS image with `Scripts/hetzner-forge.sh build-image` or `Scripts/nsc-build-and-upload-image.sh`.
|
||||
2. Recreate `burrow-forge` from the latest labeled snapshot with `Scripts/hetzner-forge.sh recreate-from-image --yes`.
|
||||
3. Run `Scripts/bootstrap-forge-intake.sh` to place the Forgejo bootstrap password file and automation SSH key under `/var/lib/burrow/intake/`.
|
||||
4. Let `burrow-forgejo-bootstrap.service` create or rotate the initial Forgejo admin account.
|
||||
3. Encrypt the Forgejo admin password and agent SSH key into `secrets/forgejo/{admin-password,agent-ssh-key}.age`.
|
||||
4. Let `burrow-forgejo-bootstrap.service` create or rotate the initial Forgejo admin account from the agenix secret path.
|
||||
5. Let `burrow-forgejo-runner-bootstrap.service` register the self-hosted Forgejo runner and seed Git identity as `agent <agent@burrow.net>`.
|
||||
6. Run `Scripts/provision-forgejo-nsc.sh` locally, re-encrypt the resulting NSC token + configs into `secrets/forgejo/*.age`, then deploy with `Scripts/forge-deploy.sh` so agenix updates the live forgejo-nsc runtime paths.
|
||||
7. Use `Scripts/cloudflare-upsert-a-record.sh` to point `git.burrow.net`, `burrow.net`, and `nsc-autoscaler.burrow.net` at the host with Cloudflare proxying disabled for ACME.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
|
||||
services.burrow.forge = {
|
||||
enable = true;
|
||||
adminPasswordFile = "/var/lib/burrow/intake/forgejo_pass_contact_at_burrow_net.txt";
|
||||
adminPasswordFile = config.age.secrets.forgejoAdminPassword.path;
|
||||
authorizedKeys = [
|
||||
(builtins.readFile ../../keys/contact_at_burrow_net.pub)
|
||||
(builtins.readFile ../../keys/agent_at_burrow_net.pub)
|
||||
|
|
@ -29,7 +29,21 @@
|
|||
|
||||
services.burrow.forgeRunner = {
|
||||
enable = true;
|
||||
sshPrivateKeyFile = "/var/lib/burrow/intake/agent_at_burrow_net_ed25519";
|
||||
sshPrivateKeyFile = config.age.secrets.forgejoAgentSshKey.path;
|
||||
};
|
||||
|
||||
age.secrets.forgejoAdminPassword = {
|
||||
file = ../../../secrets/forgejo/admin-password.age;
|
||||
mode = "0400";
|
||||
owner = "forgejo";
|
||||
group = "forgejo";
|
||||
};
|
||||
|
||||
age.secrets.forgejoAgentSshKey = {
|
||||
file = ../../../secrets/forgejo/agent-ssh-key.age;
|
||||
mode = "0400";
|
||||
owner = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
age.secrets.forgejoNscToken = {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ Burrow secrets live in `secrets/<name>.age` and are managed with `agenix`.
|
|||
|
||||
For the Forgejo Namespace Cloud runtime:
|
||||
|
||||
- `secrets/forgejo/admin-password.age`
|
||||
- `secrets/forgejo/agent-ssh-key.age`
|
||||
- `secrets/forgejo/nsc-token.age`
|
||||
- `secrets/forgejo/nsc-dispatcher-config.age`
|
||||
- `secrets/forgejo/nsc-autoscaler-config.age`
|
||||
|
|
@ -11,7 +13,8 @@ For the Forgejo Namespace Cloud runtime:
|
|||
Use:
|
||||
|
||||
- `make secret name=forgejo/nsc-token`
|
||||
- `make secret-file name=forgejo/nsc-token file=/path/to/source`
|
||||
- `make secret-file name=forgejo/agent-ssh-key file=/path/to/source`
|
||||
|
||||
The forge host decrypts these files at activation time and feeds the resulting
|
||||
paths into `services.burrow.forgejoNsc`.
|
||||
paths into `services.burrow.forge`, `services.burrow.forgeRunner`, and
|
||||
`services.burrow.forgejoNsc`.
|
||||
|
|
|
|||
11
secrets/forgejo/admin-password.age
Normal file
11
secrets/forgejo/admin-password.age
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 ux4N8Q nmGFzw38TKiVVuA9CM8wHQDVib0RddB+M/UjQnD45jk
|
||||
iZNLNBlS32zR+TNfcK27T1V3w27sFKJkWfuOzHwcOL0
|
||||
-> ssh-ed25519 IrZmAg Y53DC0wGX8mjaXkD3+jZn2DviO5iSXsnZDBNCBTmLgA
|
||||
XLz+YXzT4fYb7q0xuZMKgv88lAd0gGKaquSMcA6Yu3c
|
||||
-> ssh-ed25519 JzXUWA EDAXBKEvHccJ4KKtHjUTA+KA+wN9bBu9v+kzRTFt9AI
|
||||
JNADezBCxx26+QPD2tIpz5O8cncrJwnqaYQEWY56VGY
|
||||
--- RpjdftRPUGT80IMYKFDFuHkKEr1heJOvqrqYLufhc10
|
||||
ûÈÂ_
|
||||
F(
|
||||
((0ˆ‡Õɉ·',¿€8d]d%T[MÁ¼¬KRQÿxiIf<49>0§Òæ
|
||||
BIN
secrets/forgejo/agent-ssh-key.age
Normal file
BIN
secrets/forgejo/agent-ssh-key.age
Normal file
Binary file not shown.
|
|
@ -1,4 +1,3 @@
|
|||
{ }:
|
||||
let
|
||||
contact = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO42guJ5QvNMw3k6YKWlQnjcTsc+X4XI9F2GBtl8aHOa";
|
||||
agent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEN0+tRJy7Y2DW0uGYHb86N2t02WyU5lDNX6FaxBF/G8 agent@burrow.net";
|
||||
|
|
@ -6,6 +5,8 @@ let
|
|||
|
||||
forgeAutomation = [ contact agent forge ];
|
||||
in {
|
||||
"secrets/forgejo/admin-password.age".publicKeys = forgeAutomation;
|
||||
"secrets/forgejo/agent-ssh-key.age".publicKeys = forgeAutomation;
|
||||
"secrets/forgejo/nsc-token.age".publicKeys = forgeAutomation;
|
||||
"secrets/forgejo/nsc-dispatcher-config.age".publicKeys = forgeAutomation;
|
||||
"secrets/forgejo/nsc-autoscaler-config.age".publicKeys = forgeAutomation;
|
||||
|
|
|
|||
|
|
@ -45,6 +45,9 @@ profile. The important knobs are:
|
|||
- `namespace.machine_type` / `namespace.duration` – shape + TTL for the ephemeral
|
||||
Namespace environment. The dispatcher destroys the instance after a job so the
|
||||
TTL acts as a hard cap, not an idle timeout.
|
||||
- `namespace.linux_cache_*` / `namespace.macos_cache_*` – persistent cache
|
||||
volumes mounted into runners so Linux can keep `/nix` plus build caches warm
|
||||
and macOS can reuse Rust toolchains, Xcode package caches, and derived data.
|
||||
|
||||
### Running locally
|
||||
|
||||
|
|
@ -160,12 +163,15 @@ consume the same secret material.
|
|||
|
||||
Long-lived runtime state is now sourced from age-encrypted files:
|
||||
|
||||
- `secrets/forgejo/admin-password.age`
|
||||
- `secrets/forgejo/agent-ssh-key.age`
|
||||
- `secrets/forgejo/nsc-token.age`
|
||||
- `secrets/forgejo/nsc-dispatcher-config.age`
|
||||
- `secrets/forgejo/nsc-autoscaler-config.age`
|
||||
|
||||
After refreshing the intake files, re-encrypt them into `secrets/forgejo/*.age`
|
||||
and deploy the forge host so `config.age.secrets.*` updates the live paths for
|
||||
`services.burrow.forge`, `services.burrow.forgeRunner`, and
|
||||
`services.burrow.forgejoNsc`.
|
||||
|
||||
Run it next to the dispatcher:
|
||||
|
|
|
|||
|
|
@ -43,19 +43,23 @@ func main() {
|
|||
}
|
||||
|
||||
dispatcher, err := nsc.NewDispatcher(nsc.Options{
|
||||
BinaryPath: cfg.Namespace.NSCBinary,
|
||||
ComputeBaseURL: cfg.Namespace.ComputeBaseURL,
|
||||
DefaultImage: cfg.Namespace.Image,
|
||||
DefaultMachine: cfg.Namespace.MachineType,
|
||||
MacosBaseImageID: cfg.Namespace.MacosBaseImageID,
|
||||
MacosMachineArch: cfg.Namespace.MacosMachineArch,
|
||||
DefaultDuration: cfg.Namespace.Duration.Duration,
|
||||
WorkDir: cfg.Namespace.WorkDir,
|
||||
MaxParallel: cfg.Namespace.MaxParallel,
|
||||
RunnerNamePrefix: cfg.Runner.NamePrefix,
|
||||
Executor: cfg.Runner.Executor,
|
||||
Network: cfg.Namespace.Network,
|
||||
Logger: logger,
|
||||
BinaryPath: cfg.Namespace.NSCBinary,
|
||||
ComputeBaseURL: cfg.Namespace.ComputeBaseURL,
|
||||
DefaultImage: cfg.Namespace.Image,
|
||||
DefaultMachine: cfg.Namespace.MachineType,
|
||||
MacosBaseImageID: cfg.Namespace.MacosBaseImageID,
|
||||
MacosMachineArch: cfg.Namespace.MacosMachineArch,
|
||||
DefaultDuration: cfg.Namespace.Duration.Duration,
|
||||
WorkDir: cfg.Namespace.WorkDir,
|
||||
MaxParallel: cfg.Namespace.MaxParallel,
|
||||
LinuxCachePath: cfg.Namespace.LinuxCachePath,
|
||||
LinuxCacheVolumes: toNSCCacheVolumes(cfg.Namespace.LinuxCacheVolumes),
|
||||
MacosCachePath: cfg.Namespace.MacosCachePath,
|
||||
MacosCacheVolumes: toNSCCacheVolumes(cfg.Namespace.MacosCacheVolumes),
|
||||
RunnerNamePrefix: cfg.Runner.NamePrefix,
|
||||
Executor: cfg.Runner.Executor,
|
||||
Network: cfg.Namespace.Network,
|
||||
Logger: logger,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("failed to create dispatcher", "error", err)
|
||||
|
|
@ -88,3 +92,15 @@ func main() {
|
|||
defer cancel()
|
||||
_ = srv.Shutdown(ctx)
|
||||
}
|
||||
|
||||
func toNSCCacheVolumes(volumes []config.CacheVolumeConfig) []nsc.CacheVolume {
|
||||
out := make([]nsc.CacheVolume, 0, len(volumes))
|
||||
for _, volume := range volumes {
|
||||
out = append(out, nsc.CacheVolume{
|
||||
Tag: volume.Tag,
|
||||
MountPoint: volume.MountPoint,
|
||||
SizeGb: volume.SizeGb,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,19 @@ namespace:
|
|||
workdir: "/var/lib/forgejo-runner"
|
||||
max_parallel: 4
|
||||
network: ""
|
||||
linux_cache_path: "/var/cache/burrow"
|
||||
linux_cache_volumes:
|
||||
- tag: "burrow-forgejo-linux-nix"
|
||||
mount_point: "/nix"
|
||||
size_gb: 60
|
||||
- tag: "burrow-forgejo-linux-cache"
|
||||
mount_point: "/var/cache/burrow"
|
||||
size_gb: 40
|
||||
macos_cache_path: "/Users/runner/.cache/burrow"
|
||||
macos_cache_volumes:
|
||||
- tag: "burrow-forgejo-macos-cache"
|
||||
mount_point: "/Users/runner/.cache/burrow"
|
||||
size_gb: 60
|
||||
|
||||
runner:
|
||||
name_prefix: "nscloud-"
|
||||
|
|
|
|||
|
|
@ -31,6 +31,19 @@ namespace:
|
|||
instance_tags:
|
||||
- "burrow"
|
||||
network: ""
|
||||
linux_cache_path: "/var/cache/burrow"
|
||||
linux_cache_volumes:
|
||||
- tag: "burrow-forgejo-linux-nix"
|
||||
mount_point: "/nix"
|
||||
size_gb: 60
|
||||
- tag: "burrow-forgejo-linux-cache"
|
||||
mount_point: "/var/cache/burrow"
|
||||
size_gb: 40
|
||||
macos_cache_path: "/Users/runner/.cache/burrow"
|
||||
macos_cache_volumes:
|
||||
- tag: "burrow-forgejo-macos-cache"
|
||||
mount_point: "/Users/runner/.cache/burrow"
|
||||
size_gb: 60
|
||||
|
||||
runner:
|
||||
name_prefix: "nscloud-"
|
||||
|
|
|
|||
|
|
@ -49,8 +49,14 @@ type Config struct {
|
|||
Runner RunnerConfig `yaml:"runner"`
|
||||
}
|
||||
|
||||
type CacheVolumeConfig struct {
|
||||
Tag string `yaml:"tag"`
|
||||
MountPoint string `yaml:"mount_point"`
|
||||
SizeGb int64 `yaml:"size_gb"`
|
||||
}
|
||||
|
||||
type ForgejoConfig struct {
|
||||
BaseURL string `yaml:"base_url"`
|
||||
BaseURL string `yaml:"base_url"`
|
||||
// InstanceURL is the URL runners should use when registering with Forgejo.
|
||||
// This must be reachable from the spawned runner (e.g. the public URL like
|
||||
// https://git.burrow.net), and may differ from BaseURL (which can be a local
|
||||
|
|
@ -80,15 +86,19 @@ type NamespaceConfig struct {
|
|||
// MacosBaseImageID selects which macOS base image to use (e.g. "tahoe").
|
||||
MacosBaseImageID string `yaml:"macos_base_image_id"`
|
||||
// MacosMachineArch is the architecture used for macOS instances (typically "arm64").
|
||||
MacosMachineArch string `yaml:"macos_machine_arch"`
|
||||
Duration Duration `yaml:"duration"`
|
||||
WorkDir string `yaml:"workdir"`
|
||||
MaxParallel int64 `yaml:"max_parallel"`
|
||||
Environment []string `yaml:"environment"`
|
||||
AllowLabels []string `yaml:"allow_labels"`
|
||||
AllowScopes []string `yaml:"allow_scopes"`
|
||||
Network string `yaml:"network"`
|
||||
InstanceTags []string `yaml:"instance_tags"`
|
||||
MacosMachineArch string `yaml:"macos_machine_arch"`
|
||||
Duration Duration `yaml:"duration"`
|
||||
WorkDir string `yaml:"workdir"`
|
||||
MaxParallel int64 `yaml:"max_parallel"`
|
||||
Environment []string `yaml:"environment"`
|
||||
AllowLabels []string `yaml:"allow_labels"`
|
||||
AllowScopes []string `yaml:"allow_scopes"`
|
||||
Network string `yaml:"network"`
|
||||
InstanceTags []string `yaml:"instance_tags"`
|
||||
LinuxCachePath string `yaml:"linux_cache_path"`
|
||||
LinuxCacheVolumes []CacheVolumeConfig `yaml:"linux_cache_volumes"`
|
||||
MacosCachePath string `yaml:"macos_cache_path"`
|
||||
MacosCacheVolumes []CacheVolumeConfig `yaml:"macos_cache_volumes"`
|
||||
}
|
||||
|
||||
type RunnerConfig struct {
|
||||
|
|
@ -160,6 +170,46 @@ func (c *Config) Validate() error {
|
|||
if c.Namespace.MaxParallel <= 0 {
|
||||
c.Namespace.MaxParallel = 4
|
||||
}
|
||||
if c.Namespace.LinuxCachePath == "" {
|
||||
c.Namespace.LinuxCachePath = "/var/cache/burrow"
|
||||
}
|
||||
if len(c.Namespace.LinuxCacheVolumes) == 0 {
|
||||
c.Namespace.LinuxCacheVolumes = []CacheVolumeConfig{
|
||||
{
|
||||
Tag: "burrow-forgejo-linux-nix",
|
||||
MountPoint: "/nix",
|
||||
SizeGb: 60,
|
||||
},
|
||||
{
|
||||
Tag: "burrow-forgejo-linux-cache",
|
||||
MountPoint: c.Namespace.LinuxCachePath,
|
||||
SizeGb: 40,
|
||||
},
|
||||
}
|
||||
}
|
||||
if c.Namespace.MacosCachePath == "" {
|
||||
c.Namespace.MacosCachePath = "/Users/runner/.cache/burrow"
|
||||
}
|
||||
if len(c.Namespace.MacosCacheVolumes) == 0 {
|
||||
c.Namespace.MacosCacheVolumes = []CacheVolumeConfig{
|
||||
{
|
||||
Tag: "burrow-forgejo-macos-cache",
|
||||
MountPoint: c.Namespace.MacosCachePath,
|
||||
SizeGb: 60,
|
||||
},
|
||||
}
|
||||
}
|
||||
for _, volume := range append(append([]CacheVolumeConfig{}, c.Namespace.LinuxCacheVolumes...), c.Namespace.MacosCacheVolumes...) {
|
||||
if strings.TrimSpace(volume.Tag) == "" {
|
||||
return errors.New("namespace cache volume tag is required")
|
||||
}
|
||||
if strings.TrimSpace(volume.MountPoint) == "" {
|
||||
return fmt.Errorf("namespace cache volume %q mount_point is required", volume.Tag)
|
||||
}
|
||||
if volume.SizeGb <= 0 {
|
||||
return fmt.Errorf("namespace cache volume %q size_gb must be positive", volume.Tag)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,19 +17,29 @@ import (
|
|||
)
|
||||
|
||||
type Options struct {
|
||||
BinaryPath string
|
||||
DefaultImage string
|
||||
DefaultMachine string
|
||||
DefaultDuration time.Duration
|
||||
WorkDir string
|
||||
MaxParallel int64
|
||||
RunnerNamePrefix string
|
||||
Executor string
|
||||
Network string
|
||||
ComputeBaseURL string
|
||||
MacosBaseImageID string
|
||||
MacosMachineArch string
|
||||
Logger *slog.Logger
|
||||
BinaryPath string
|
||||
DefaultImage string
|
||||
DefaultMachine string
|
||||
DefaultDuration time.Duration
|
||||
WorkDir string
|
||||
MaxParallel int64
|
||||
RunnerNamePrefix string
|
||||
Executor string
|
||||
Network string
|
||||
ComputeBaseURL string
|
||||
MacosBaseImageID string
|
||||
MacosMachineArch string
|
||||
LinuxCachePath string
|
||||
LinuxCacheVolumes []CacheVolume
|
||||
MacosCachePath string
|
||||
MacosCacheVolumes []CacheVolume
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type CacheVolume struct {
|
||||
Tag string
|
||||
MountPoint string
|
||||
SizeGb int64
|
||||
}
|
||||
|
||||
type LaunchRequest struct {
|
||||
|
|
@ -73,6 +83,12 @@ func NewDispatcher(opts Options) (*Dispatcher, error) {
|
|||
if opts.DefaultDuration == 0 {
|
||||
opts.DefaultDuration = 30 * time.Minute
|
||||
}
|
||||
if opts.LinuxCachePath == "" {
|
||||
opts.LinuxCachePath = "/var/cache/burrow"
|
||||
}
|
||||
if opts.MacosCachePath == "" {
|
||||
opts.MacosCachePath = "/Users/runner/.cache/burrow"
|
||||
}
|
||||
logger := opts.Logger
|
||||
if logger == nil {
|
||||
logger = slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
|
|
@ -104,6 +120,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
|
|||
}
|
||||
machineType := choose(req.MachineType, d.opts.DefaultMachine)
|
||||
image := choose(req.Image, d.opts.DefaultImage)
|
||||
if req.ExtraEnv == nil {
|
||||
req.ExtraEnv = make(map[string]string)
|
||||
}
|
||||
|
||||
if hasWindowsLabel(req.Labels) {
|
||||
if err := d.launchWindowsRunnerViaWinRM(ctx, runnerName, req, duration, machineType); err != nil {
|
||||
|
|
@ -113,6 +132,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
|
|||
}
|
||||
|
||||
if hasMacOSLabel(req.Labels) {
|
||||
if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok {
|
||||
req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.MacosCachePath
|
||||
}
|
||||
// Compute macOS shapes differ from the Linux "run" defaults. If the request
|
||||
// didn't specify a machine type, ensure we pick a macOS-valid default.
|
||||
if machineType == "" || machineType == d.opts.DefaultMachine {
|
||||
|
|
@ -129,6 +151,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
|
|||
}
|
||||
return runnerName, nil
|
||||
}
|
||||
if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok {
|
||||
req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.LinuxCachePath
|
||||
}
|
||||
|
||||
env := map[string]string{
|
||||
"FORGEJO_INSTANCE_URL": req.InstanceURL,
|
||||
|
|
@ -140,9 +165,6 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
|
|||
for k, v := range req.ExtraEnv {
|
||||
env[k] = v
|
||||
}
|
||||
if _, ok := env["NSC_CACHE_PATH"]; !ok {
|
||||
env["NSC_CACHE_PATH"] = "/nix/store"
|
||||
}
|
||||
|
||||
script := d.bootstrapScript()
|
||||
args := []string{
|
||||
|
|
@ -161,6 +183,7 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
|
|||
if d.opts.Network != "" {
|
||||
args = append(args, "--network", d.opts.Network)
|
||||
}
|
||||
args = appendVolumeArgs(args, d.opts.LinuxCacheVolumes)
|
||||
for key, value := range env {
|
||||
if value == "" {
|
||||
continue
|
||||
|
|
@ -370,6 +393,16 @@ func choose(values ...string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
func appendVolumeArgs(args []string, volumes []CacheVolume) []string {
|
||||
for _, volume := range volumes {
|
||||
if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 {
|
||||
continue
|
||||
}
|
||||
args = append(args, "--volume", fmt.Sprintf("cache:%s:%s:%d", volume.Tag, volume.MountPoint, volume.SizeGb))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func (d *Dispatcher) bootstrapScript() string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(`set -euo pipefail
|
||||
|
|
|
|||
|
|
@ -206,12 +206,8 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r
|
|||
for k, v := range req.ExtraEnv {
|
||||
env[k] = v
|
||||
}
|
||||
// Best-effort caching: workflows call Scripts/nscloud-cache.sh, which is a
|
||||
// no-op unless NSC_CACHE_PATH is set. This may still be skipped if spacectl
|
||||
// lacks credentials, but setting the path is harmless and keeps behavior
|
||||
// consistent across macOS / Linux runners.
|
||||
if _, ok := env["NSC_CACHE_PATH"]; !ok {
|
||||
env["NSC_CACHE_PATH"] = "/Users/runner/.cache/nscloud"
|
||||
env["NSC_CACHE_PATH"] = d.opts.MacosCachePath
|
||||
}
|
||||
|
||||
deadline := timestamppb.New(time.Now().Add(ttl))
|
||||
|
|
@ -243,10 +239,15 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r
|
|||
},
|
||||
},
|
||||
}
|
||||
experimental := &computev1beta.CreateInstanceRequest_ExperimentalFeatures{}
|
||||
if imageID := macosComputeBaseImageID(d.opts.MacosBaseImageID); imageID != "" {
|
||||
createReq.Experimental = &computev1beta.CreateInstanceRequest_ExperimentalFeatures{
|
||||
MacosBaseImageId: imageID,
|
||||
}
|
||||
experimental.MacosBaseImageId = imageID
|
||||
}
|
||||
if volumes := computeCacheVolumeRequests(d.opts.MacosCacheVolumes); len(volumes) > 0 {
|
||||
experimental.Volumes = volumes
|
||||
}
|
||||
if experimental.MacosBaseImageId != "" || len(experimental.Volumes) > 0 {
|
||||
createReq.Experimental = experimental
|
||||
}
|
||||
|
||||
d.log.Info("launching Namespace macos runner",
|
||||
|
|
@ -572,6 +573,22 @@ func (d *Dispatcher) destroyComputeInstance(ctx context.Context, client computev
|
|||
d.log.Info("macos runner destroyed", "runner", runnerName, "instance", instanceID)
|
||||
}
|
||||
|
||||
func computeCacheVolumeRequests(volumes []CacheVolume) []*computev1beta.VolumeRequest {
|
||||
var out []*computev1beta.VolumeRequest
|
||||
for _, volume := range volumes {
|
||||
if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, &computev1beta.VolumeRequest{
|
||||
MountPoint: volume.MountPoint,
|
||||
Tag: volume.Tag,
|
||||
SizeMb: volume.SizeGb * 1024,
|
||||
PersistencyKind: computev1beta.VolumeRequest_CACHE,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func macosBootstrapScript() string {
|
||||
// Keep this script self-contained: it runs on a fresh macOS VM base image.
|
||||
var b strings.Builder
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ func (d *Dispatcher) launchMacOSRunnerViaNSC(ctx context.Context, runnerName str
|
|||
"--wait_timeout", a.waitTimeout.String(),
|
||||
}
|
||||
args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
|
||||
args = appendVolumeArgs(args, d.opts.MacosCacheVolumes)
|
||||
|
||||
createCtx, cancel := context.WithTimeout(ctx, a.createTimeout)
|
||||
defer cancel()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue