From ed247b2f5e55abb93e0c18eaa8cd9a4606ee1b99 Mon Sep 17 00:00:00 2001 From: Conrad Kramer Date: Thu, 19 Mar 2026 00:04:27 -0700 Subject: [PATCH] Wire runner caches and forge secrets through agenix --- .forgejo/workflows/build-apple.yml | 33 +++++++-- .forgejo/workflows/build-rust.yml | 19 ++++- .forgejo/workflows/build-site.yml | 8 ++ .../NetworkExtension/libburrow/build-rust.sh | 8 +- Makefile | 4 +- flake.nix | 1 + nixos/README.md | 8 +- nixos/hosts/burrow-forge/default.nix | 18 ++++- secrets/README.md | 7 +- secrets/forgejo/admin-password.age | 11 +++ secrets/forgejo/agent-ssh-key.age | Bin 0 -> 843 bytes secrets/secrets.nix | 3 +- services/forgejo-nsc/README.md | 6 ++ .../cmd/forgejo-nsc-dispatcher/main.go | 42 +++++++---- services/forgejo-nsc/config.example.yaml | 13 ++++ services/forgejo-nsc/deploy/dispatcher.yaml | 13 ++++ .../forgejo-nsc/internal/config/config.go | 70 +++++++++++++++--- .../forgejo-nsc/internal/nsc/dispatcher.go | 65 ++++++++++++---- services/forgejo-nsc/internal/nsc/macos.go | 33 +++++++-- .../forgejo-nsc/internal/nsc/macos_nsc.go | 1 + 20 files changed, 299 insertions(+), 64 deletions(-) create mode 100644 secrets/forgejo/admin-password.age create mode 100644 secrets/forgejo/agent-ssh-key.age diff --git a/.forgejo/workflows/build-apple.yml b/.forgejo/workflows/build-apple.yml index 9b7fcc8..d55957e 100644 --- a/.forgejo/workflows/build-apple.yml +++ b/.forgejo/workflows/build-apple.yml @@ -22,14 +22,17 @@ jobs: matrix: include: - platform: macOS + cache-id: macos destination: platform=macOS rust-targets: x86_64-apple-darwin,aarch64-apple-darwin - platform: iOS Simulator + cache-id: ios-simulator destination: platform=iOS Simulator,name=iPhone 17 Pro rust-targets: aarch64-apple-ios-sim,x86_64-apple-ios env: CARGO_INCREMENTAL: 0 RUST_BACKTRACE: short + RUSTC_WRAPPER: sccache steps: - name: Checkout uses: https://code.forgejo.org/actions/checkout@v4 @@ -65,12 +68,29 @@ jobs: echo "DEVELOPER_DIR=$selected" >> "$GITHUB_ENV" DEVELOPER_DIR="$selected" /usr/bin/xcodebuild -version || true + - name: Prepare Cache Dirs + shell: bash + run: | + set -euo pipefail + cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}" + mkdir -p \ + "${cache_root}/cargo" \ + "${cache_root}/rustup" \ + "${cache_root}/sccache" \ + "${cache_root}/apple/PackageCache" \ + "${cache_root}/apple/SourcePackages" \ + "${cache_root}/apple/DerivedData/${{ matrix.cache-id }}" + echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}" + echo "RUSTUP_HOME=${cache_root}/rustup" >> "${GITHUB_ENV}" + echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}" + echo "APPLE_PACKAGE_CACHE=${cache_root}/apple/PackageCache" >> "${GITHUB_ENV}" + echo "APPLE_SOURCE_PACKAGES=${cache_root}/apple/SourcePackages" >> "${GITHUB_ENV}" + echo "APPLE_DERIVED_DATA=${cache_root}/apple/DerivedData/${{ matrix.cache-id }}" >> "${GITHUB_ENV}" + - name: Install Rust shell: bash run: | set -euo pipefail - export RUSTUP_HOME="${HOME}/.rustup" - export CARGO_HOME="${HOME}/.cargo" if ! command -v rustup >/dev/null 2>&1; then curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.85.0 @@ -98,6 +118,9 @@ jobs: if ! command -v protoc >/dev/null 2>&1; then brew install protobuf fi + if ! command -v sccache >/dev/null 2>&1; then + brew install sccache + fi - name: Build shell: bash @@ -111,9 +134,9 @@ jobs: -skipPackagePluginValidation \ -skipMacroValidation \ -onlyUsePackageVersionsFromResolvedFile \ - -clonedSourcePackagesDirPath SourcePackages \ - -packageCachePath "$PWD/PackageCache" \ - -derivedDataPath "$PWD/DerivedData" \ + -clonedSourcePackagesDirPath "$APPLE_SOURCE_PACKAGES" \ + -packageCachePath "$APPLE_PACKAGE_CACHE" \ + -derivedDataPath "$APPLE_DERIVED_DATA" \ CODE_SIGNING_ALLOWED=NO \ CODE_SIGNING_REQUIRED=NO \ CODE_SIGN_IDENTITY="" \ diff --git a/.forgejo/workflows/build-rust.yml b/.forgejo/workflows/build-rust.yml index 2df1ad3..7fd2667 100644 --- a/.forgejo/workflows/build-rust.yml +++ b/.forgejo/workflows/build-rust.yml @@ -17,6 +17,10 @@ jobs: rust: name: Cargo Test runs-on: [self-hosted, linux, x86_64, burrow-forge] + env: + CARGO_INCREMENTAL: 0 + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 20G steps: - name: Checkout uses: https://code.forgejo.org/actions/checkout@v4 @@ -24,8 +28,21 @@ jobs: token: ${{ github.token }} fetch-depth: 0 + - name: Prepare Cache Dirs + shell: bash + run: | + set -euo pipefail + cache_root="${HOME}/.cache/burrow" + mkdir -p "${cache_root}/cargo" "${cache_root}/sccache" + echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}" + echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}" + - name: Test shell: bash run: | set -euo pipefail - nix develop .#ci -c cargo test --workspace --all-features + nix develop .#ci -c bash -lc ' + sccache --zero-stats >/dev/null 2>&1 || true + cargo test --workspace --all-features + sccache --show-stats || true + ' diff --git a/.forgejo/workflows/build-site.yml b/.forgejo/workflows/build-site.yml index 6f7c5e2..de296d4 100644 --- a/.forgejo/workflows/build-site.yml +++ b/.forgejo/workflows/build-site.yml @@ -24,6 +24,14 @@ jobs: token: ${{ github.token }} fetch-depth: 0 + - name: Prepare Cache Dirs + shell: bash + run: | + set -euo pipefail + cache_root="${HOME}/.cache/burrow" + mkdir -p "${cache_root}/npm" + echo "NPM_CONFIG_CACHE=${cache_root}/npm" >> "${GITHUB_ENV}" + - name: Build shell: bash run: | diff --git a/Apple/NetworkExtension/libburrow/build-rust.sh b/Apple/NetworkExtension/libburrow/build-rust.sh index 6f455a9..258351c 100755 --- a/Apple/NetworkExtension/libburrow/build-rust.sh +++ b/Apple/NetworkExtension/libburrow/build-rust.sh @@ -73,7 +73,13 @@ CARGO_PATH="$(dirname $PROTOC):$CARGO_PATH" # Run cargo without the various environment variables set by Xcode. # Those variables can confuse cargo and the build scripts it runs. -env -i PATH="$CARGO_PATH" PROTOC="$PROTOC" CARGO_TARGET_DIR="${CONFIGURATION_TEMP_DIR}/target" IPHONEOS_DEPLOYMENT_TARGET="$IPHONEOS_DEPLOYMENT_TARGET" MACOSX_DEPLOYMENT_TARGET="$MACOSX_DEPLOYMENT_TARGET" cargo build "${CARGO_ARGS[@]}" +EXTRA_ENV=() +for VAR_NAME in HOME CARGO_HOME RUSTUP_HOME RUSTC_WRAPPER SCCACHE_DIR CARGO_INCREMENTAL; do + if [[ -n "${!VAR_NAME:-}" ]]; then + EXTRA_ENV+=("${VAR_NAME}=${!VAR_NAME}") + fi +done +env -i PATH="$CARGO_PATH" PROTOC="$PROTOC" CARGO_TARGET_DIR="${CONFIGURATION_TEMP_DIR}/target" IPHONEOS_DEPLOYMENT_TARGET="$IPHONEOS_DEPLOYMENT_TARGET" MACOSX_DEPLOYMENT_TARGET="$MACOSX_DEPLOYMENT_TARGET" "${EXTRA_ENV[@]}" cargo build "${CARGO_ARGS[@]}" mkdir -p "${BUILT_PRODUCTS_DIR}" diff --git a/Makefile b/Makefile index e852e32..1f15f36 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ FLAKE ?= . AGENIX ?= nix run ${FLAKE}\#agenix -- -SECRETS := forgejo/nsc-token \ +SECRETS := forgejo/admin-password \ + forgejo/agent-ssh-key \ + forgejo/nsc-token \ forgejo/nsc-dispatcher-config \ forgejo/nsc-autoscaler-config diff --git a/flake.nix b/flake.nix index 51e4bc9..ed59619 100644 --- a/flake.nix +++ b/flake.nix @@ -36,6 +36,7 @@ agenixPkg = agenix.packages.${system}.agenix; commonPackages = with pkgs; [ cargo + sccache rustc rustfmt clippy diff --git a/nixos/README.md b/nixos/README.md index f37637c..aa0fff6 100644 --- a/nixos/README.md +++ b/nixos/README.md @@ -15,19 +15,19 @@ Mail hosting is intentionally not part of this NixOS host in the current plan. B - `keys/agent_at_burrow_net.pub`: automation SSH public key - `../Scripts/hetzner-forge.sh`: Hetzner inventory and replace workflow - `../Scripts/nsc-build-and-upload-image.sh`: temporary Namespace builder -> raw image -> Hetzner snapshot -- `../Scripts/bootstrap-forge-intake.sh`: copy the Forgejo bootstrap password and agent SSH key into `/var/lib/burrow/intake/` +- `../Scripts/bootstrap-forge-intake.sh`: legacy intake bootstrap helper; current forge runtime secrets should live in `../secrets/forgejo/*.age` - `../Scripts/check-forge-host.sh`: verify Forgejo, Caddy, the local runner, and optional NSC services after boot - `../Scripts/cloudflare-upsert-a-record.sh`: upsert DNS-only Cloudflare `A` records for Burrow host cutovers - `../Scripts/forge-deploy.sh`: remote `nixos-rebuild` entrypoint for the forge host - `../Scripts/provision-forgejo-nsc.sh`: render Burrow Namespace dispatcher/autoscaler bootstrap inputs and ensure the default Forgejo scope exists -- `../secrets/forgejo/*.age`: authoritative encrypted Namespace token + dispatcher/autoscaler configs for the forge host +- `../secrets/forgejo/*.age`: authoritative encrypted forge admin password, agent SSH key, and Namespace runtime configs for the forge host ## Intended Flow 1. Build and upload the raw NixOS image with `Scripts/hetzner-forge.sh build-image` or `Scripts/nsc-build-and-upload-image.sh`. 2. Recreate `burrow-forge` from the latest labeled snapshot with `Scripts/hetzner-forge.sh recreate-from-image --yes`. -3. Run `Scripts/bootstrap-forge-intake.sh` to place the Forgejo bootstrap password file and automation SSH key under `/var/lib/burrow/intake/`. -4. Let `burrow-forgejo-bootstrap.service` create or rotate the initial Forgejo admin account. +3. Encrypt the Forgejo admin password and agent SSH key into `secrets/forgejo/{admin-password,agent-ssh-key}.age`. +4. Let `burrow-forgejo-bootstrap.service` create or rotate the initial Forgejo admin account from the agenix secret path. 5. Let `burrow-forgejo-runner-bootstrap.service` register the self-hosted Forgejo runner and seed Git identity as `agent `. 6. Run `Scripts/provision-forgejo-nsc.sh` locally, re-encrypt the resulting NSC token + configs into `secrets/forgejo/*.age`, then deploy with `Scripts/forge-deploy.sh` so agenix updates the live forgejo-nsc runtime paths. 7. Use `Scripts/cloudflare-upsert-a-record.sh` to point `git.burrow.net`, `burrow.net`, and `nsc-autoscaler.burrow.net` at the host with Cloudflare proxying disabled for ACME. diff --git a/nixos/hosts/burrow-forge/default.nix b/nixos/hosts/burrow-forge/default.nix index 7dc828d..0ce7964 100644 --- a/nixos/hosts/burrow-forge/default.nix +++ b/nixos/hosts/burrow-forge/default.nix @@ -20,7 +20,7 @@ services.burrow.forge = { enable = true; - adminPasswordFile = "/var/lib/burrow/intake/forgejo_pass_contact_at_burrow_net.txt"; + adminPasswordFile = config.age.secrets.forgejoAdminPassword.path; authorizedKeys = [ (builtins.readFile ../../keys/contact_at_burrow_net.pub) (builtins.readFile ../../keys/agent_at_burrow_net.pub) @@ -29,7 +29,21 @@ services.burrow.forgeRunner = { enable = true; - sshPrivateKeyFile = "/var/lib/burrow/intake/agent_at_burrow_net_ed25519"; + sshPrivateKeyFile = config.age.secrets.forgejoAgentSshKey.path; + }; + + age.secrets.forgejoAdminPassword = { + file = ../../../secrets/forgejo/admin-password.age; + mode = "0400"; + owner = "forgejo"; + group = "forgejo"; + }; + + age.secrets.forgejoAgentSshKey = { + file = ../../../secrets/forgejo/agent-ssh-key.age; + mode = "0400"; + owner = "root"; + group = "root"; }; age.secrets.forgejoNscToken = { diff --git a/secrets/README.md b/secrets/README.md index 2132079..f7d67f5 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -4,6 +4,8 @@ Burrow secrets live in `secrets/.age` and are managed with `agenix`. For the Forgejo Namespace Cloud runtime: +- `secrets/forgejo/admin-password.age` +- `secrets/forgejo/agent-ssh-key.age` - `secrets/forgejo/nsc-token.age` - `secrets/forgejo/nsc-dispatcher-config.age` - `secrets/forgejo/nsc-autoscaler-config.age` @@ -11,7 +13,8 @@ For the Forgejo Namespace Cloud runtime: Use: - `make secret name=forgejo/nsc-token` -- `make secret-file name=forgejo/nsc-token file=/path/to/source` +- `make secret-file name=forgejo/agent-ssh-key file=/path/to/source` The forge host decrypts these files at activation time and feeds the resulting -paths into `services.burrow.forgejoNsc`. +paths into `services.burrow.forge`, `services.burrow.forgeRunner`, and +`services.burrow.forgejoNsc`. diff --git a/secrets/forgejo/admin-password.age b/secrets/forgejo/admin-password.age new file mode 100644 index 0000000..53cfa83 --- /dev/null +++ b/secrets/forgejo/admin-password.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 ux4N8Q nmGFzw38TKiVVuA9CM8wHQDVib0RddB+M/UjQnD45jk +iZNLNBlS32zR+TNfcK27T1V3w27sFKJkWfuOzHwcOL0 +-> ssh-ed25519 IrZmAg Y53DC0wGX8mjaXkD3+jZn2DviO5iSXsnZDBNCBTmLgA +XLz+YXzT4fYb7q0xuZMKgv88lAd0gGKaquSMcA6Yu3c +-> ssh-ed25519 JzXUWA EDAXBKEvHccJ4KKtHjUTA+KA+wN9bBu9v+kzRTFt9AI +JNADezBCxx26+QPD2tIpz5O8cncrJwnqaYQEWY56VGY +--- RpjdftRPUGT80IMYKFDFuHkKEr1heJOvqrqYLufhc10 +_ +F( +((0ɉ',8d]d%T[MKRQxiIf0 \ No newline at end of file diff --git a/secrets/forgejo/agent-ssh-key.age b/secrets/forgejo/agent-ssh-key.age new file mode 100644 index 0000000000000000000000000000000000000000..44ce1141a76201c003a1d4f3f4ccae7c462780cf GIT binary patch literal 843 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCSHtuXPk2vkVP2~YD* zPR#K2DbMyODlsc^Eh_VmaI1(iGxT?JGjpl(%FGM*tnf+-3FS&N&h~f8smu+|D$y@D z4RK9Oi7Gd)aw~Pr3~>pMN)HdriE_*^O!Y7+DMq)=vnVRpF7D!p2SSmMb1`|FARrQ?2^sGK2b(BF3|e zk{bM(pJeT-e-Sd5`72y(PJzt-MiD&g2@9scysgeB$Gi96AP-aC%uub<2PTfbU3$vJ$rk4~4 z8A1jN&zjq{Yi<0P5VOndR!c&0-?#Z56SX?}ye4`Eaw~1?E11G(_HRkb-}B1+-Dh7l zc_gb{Ugf=ZKPTsgOUz7#dJ8|C_nf!Oa*mhpM=c|X(DMe9)K4tg@q2k}N6)F2gWLb~ O%k~Gp&1H(7mJI+SI9>_> literal 0 HcmV?d00001 diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 1cacc6a..9d40bf3 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -1,4 +1,3 @@ -{ }: let contact = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO42guJ5QvNMw3k6YKWlQnjcTsc+X4XI9F2GBtl8aHOa"; agent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEN0+tRJy7Y2DW0uGYHb86N2t02WyU5lDNX6FaxBF/G8 agent@burrow.net"; @@ -6,6 +5,8 @@ let forgeAutomation = [ contact agent forge ]; in { + "secrets/forgejo/admin-password.age".publicKeys = forgeAutomation; + "secrets/forgejo/agent-ssh-key.age".publicKeys = forgeAutomation; "secrets/forgejo/nsc-token.age".publicKeys = forgeAutomation; "secrets/forgejo/nsc-dispatcher-config.age".publicKeys = forgeAutomation; "secrets/forgejo/nsc-autoscaler-config.age".publicKeys = forgeAutomation; diff --git a/services/forgejo-nsc/README.md b/services/forgejo-nsc/README.md index 2cffe63..5b2926b 100644 --- a/services/forgejo-nsc/README.md +++ b/services/forgejo-nsc/README.md @@ -45,6 +45,9 @@ profile. The important knobs are: - `namespace.machine_type` / `namespace.duration` – shape + TTL for the ephemeral Namespace environment. The dispatcher destroys the instance after a job so the TTL acts as a hard cap, not an idle timeout. +- `namespace.linux_cache_*` / `namespace.macos_cache_*` – persistent cache + volumes mounted into runners so Linux can keep `/nix` plus build caches warm + and macOS can reuse Rust toolchains, Xcode package caches, and derived data. ### Running locally @@ -160,12 +163,15 @@ consume the same secret material. Long-lived runtime state is now sourced from age-encrypted files: +- `secrets/forgejo/admin-password.age` +- `secrets/forgejo/agent-ssh-key.age` - `secrets/forgejo/nsc-token.age` - `secrets/forgejo/nsc-dispatcher-config.age` - `secrets/forgejo/nsc-autoscaler-config.age` After refreshing the intake files, re-encrypt them into `secrets/forgejo/*.age` and deploy the forge host so `config.age.secrets.*` updates the live paths for +`services.burrow.forge`, `services.burrow.forgeRunner`, and `services.burrow.forgejoNsc`. Run it next to the dispatcher: diff --git a/services/forgejo-nsc/cmd/forgejo-nsc-dispatcher/main.go b/services/forgejo-nsc/cmd/forgejo-nsc-dispatcher/main.go index 9dcbfb1..3a04a26 100644 --- a/services/forgejo-nsc/cmd/forgejo-nsc-dispatcher/main.go +++ b/services/forgejo-nsc/cmd/forgejo-nsc-dispatcher/main.go @@ -43,19 +43,23 @@ func main() { } dispatcher, err := nsc.NewDispatcher(nsc.Options{ - BinaryPath: cfg.Namespace.NSCBinary, - ComputeBaseURL: cfg.Namespace.ComputeBaseURL, - DefaultImage: cfg.Namespace.Image, - DefaultMachine: cfg.Namespace.MachineType, - MacosBaseImageID: cfg.Namespace.MacosBaseImageID, - MacosMachineArch: cfg.Namespace.MacosMachineArch, - DefaultDuration: cfg.Namespace.Duration.Duration, - WorkDir: cfg.Namespace.WorkDir, - MaxParallel: cfg.Namespace.MaxParallel, - RunnerNamePrefix: cfg.Runner.NamePrefix, - Executor: cfg.Runner.Executor, - Network: cfg.Namespace.Network, - Logger: logger, + BinaryPath: cfg.Namespace.NSCBinary, + ComputeBaseURL: cfg.Namespace.ComputeBaseURL, + DefaultImage: cfg.Namespace.Image, + DefaultMachine: cfg.Namespace.MachineType, + MacosBaseImageID: cfg.Namespace.MacosBaseImageID, + MacosMachineArch: cfg.Namespace.MacosMachineArch, + DefaultDuration: cfg.Namespace.Duration.Duration, + WorkDir: cfg.Namespace.WorkDir, + MaxParallel: cfg.Namespace.MaxParallel, + LinuxCachePath: cfg.Namespace.LinuxCachePath, + LinuxCacheVolumes: toNSCCacheVolumes(cfg.Namespace.LinuxCacheVolumes), + MacosCachePath: cfg.Namespace.MacosCachePath, + MacosCacheVolumes: toNSCCacheVolumes(cfg.Namespace.MacosCacheVolumes), + RunnerNamePrefix: cfg.Runner.NamePrefix, + Executor: cfg.Runner.Executor, + Network: cfg.Namespace.Network, + Logger: logger, }) if err != nil { logger.Error("failed to create dispatcher", "error", err) @@ -88,3 +92,15 @@ func main() { defer cancel() _ = srv.Shutdown(ctx) } + +func toNSCCacheVolumes(volumes []config.CacheVolumeConfig) []nsc.CacheVolume { + out := make([]nsc.CacheVolume, 0, len(volumes)) + for _, volume := range volumes { + out = append(out, nsc.CacheVolume{ + Tag: volume.Tag, + MountPoint: volume.MountPoint, + SizeGb: volume.SizeGb, + }) + } + return out +} diff --git a/services/forgejo-nsc/config.example.yaml b/services/forgejo-nsc/config.example.yaml index 5dc7551..fcd56ec 100644 --- a/services/forgejo-nsc/config.example.yaml +++ b/services/forgejo-nsc/config.example.yaml @@ -21,6 +21,19 @@ namespace: workdir: "/var/lib/forgejo-runner" max_parallel: 4 network: "" + linux_cache_path: "/var/cache/burrow" + linux_cache_volumes: + - tag: "burrow-forgejo-linux-nix" + mount_point: "/nix" + size_gb: 60 + - tag: "burrow-forgejo-linux-cache" + mount_point: "/var/cache/burrow" + size_gb: 40 + macos_cache_path: "/Users/runner/.cache/burrow" + macos_cache_volumes: + - tag: "burrow-forgejo-macos-cache" + mount_point: "/Users/runner/.cache/burrow" + size_gb: 60 runner: name_prefix: "nscloud-" diff --git a/services/forgejo-nsc/deploy/dispatcher.yaml b/services/forgejo-nsc/deploy/dispatcher.yaml index 1e45d39..b906b75 100644 --- a/services/forgejo-nsc/deploy/dispatcher.yaml +++ b/services/forgejo-nsc/deploy/dispatcher.yaml @@ -31,6 +31,19 @@ namespace: instance_tags: - "burrow" network: "" + linux_cache_path: "/var/cache/burrow" + linux_cache_volumes: + - tag: "burrow-forgejo-linux-nix" + mount_point: "/nix" + size_gb: 60 + - tag: "burrow-forgejo-linux-cache" + mount_point: "/var/cache/burrow" + size_gb: 40 + macos_cache_path: "/Users/runner/.cache/burrow" + macos_cache_volumes: + - tag: "burrow-forgejo-macos-cache" + mount_point: "/Users/runner/.cache/burrow" + size_gb: 60 runner: name_prefix: "nscloud-" diff --git a/services/forgejo-nsc/internal/config/config.go b/services/forgejo-nsc/internal/config/config.go index 264cbd0..6a93e69 100644 --- a/services/forgejo-nsc/internal/config/config.go +++ b/services/forgejo-nsc/internal/config/config.go @@ -49,8 +49,14 @@ type Config struct { Runner RunnerConfig `yaml:"runner"` } +type CacheVolumeConfig struct { + Tag string `yaml:"tag"` + MountPoint string `yaml:"mount_point"` + SizeGb int64 `yaml:"size_gb"` +} + type ForgejoConfig struct { - BaseURL string `yaml:"base_url"` + BaseURL string `yaml:"base_url"` // InstanceURL is the URL runners should use when registering with Forgejo. // This must be reachable from the spawned runner (e.g. the public URL like // https://git.burrow.net), and may differ from BaseURL (which can be a local @@ -80,15 +86,19 @@ type NamespaceConfig struct { // MacosBaseImageID selects which macOS base image to use (e.g. "tahoe"). MacosBaseImageID string `yaml:"macos_base_image_id"` // MacosMachineArch is the architecture used for macOS instances (typically "arm64"). - MacosMachineArch string `yaml:"macos_machine_arch"` - Duration Duration `yaml:"duration"` - WorkDir string `yaml:"workdir"` - MaxParallel int64 `yaml:"max_parallel"` - Environment []string `yaml:"environment"` - AllowLabels []string `yaml:"allow_labels"` - AllowScopes []string `yaml:"allow_scopes"` - Network string `yaml:"network"` - InstanceTags []string `yaml:"instance_tags"` + MacosMachineArch string `yaml:"macos_machine_arch"` + Duration Duration `yaml:"duration"` + WorkDir string `yaml:"workdir"` + MaxParallel int64 `yaml:"max_parallel"` + Environment []string `yaml:"environment"` + AllowLabels []string `yaml:"allow_labels"` + AllowScopes []string `yaml:"allow_scopes"` + Network string `yaml:"network"` + InstanceTags []string `yaml:"instance_tags"` + LinuxCachePath string `yaml:"linux_cache_path"` + LinuxCacheVolumes []CacheVolumeConfig `yaml:"linux_cache_volumes"` + MacosCachePath string `yaml:"macos_cache_path"` + MacosCacheVolumes []CacheVolumeConfig `yaml:"macos_cache_volumes"` } type RunnerConfig struct { @@ -160,6 +170,46 @@ func (c *Config) Validate() error { if c.Namespace.MaxParallel <= 0 { c.Namespace.MaxParallel = 4 } + if c.Namespace.LinuxCachePath == "" { + c.Namespace.LinuxCachePath = "/var/cache/burrow" + } + if len(c.Namespace.LinuxCacheVolumes) == 0 { + c.Namespace.LinuxCacheVolumes = []CacheVolumeConfig{ + { + Tag: "burrow-forgejo-linux-nix", + MountPoint: "/nix", + SizeGb: 60, + }, + { + Tag: "burrow-forgejo-linux-cache", + MountPoint: c.Namespace.LinuxCachePath, + SizeGb: 40, + }, + } + } + if c.Namespace.MacosCachePath == "" { + c.Namespace.MacosCachePath = "/Users/runner/.cache/burrow" + } + if len(c.Namespace.MacosCacheVolumes) == 0 { + c.Namespace.MacosCacheVolumes = []CacheVolumeConfig{ + { + Tag: "burrow-forgejo-macos-cache", + MountPoint: c.Namespace.MacosCachePath, + SizeGb: 60, + }, + } + } + for _, volume := range append(append([]CacheVolumeConfig{}, c.Namespace.LinuxCacheVolumes...), c.Namespace.MacosCacheVolumes...) { + if strings.TrimSpace(volume.Tag) == "" { + return errors.New("namespace cache volume tag is required") + } + if strings.TrimSpace(volume.MountPoint) == "" { + return fmt.Errorf("namespace cache volume %q mount_point is required", volume.Tag) + } + if volume.SizeGb <= 0 { + return fmt.Errorf("namespace cache volume %q size_gb must be positive", volume.Tag) + } + } return nil } diff --git a/services/forgejo-nsc/internal/nsc/dispatcher.go b/services/forgejo-nsc/internal/nsc/dispatcher.go index 3db2481..7fa6d62 100644 --- a/services/forgejo-nsc/internal/nsc/dispatcher.go +++ b/services/forgejo-nsc/internal/nsc/dispatcher.go @@ -17,19 +17,29 @@ import ( ) type Options struct { - BinaryPath string - DefaultImage string - DefaultMachine string - DefaultDuration time.Duration - WorkDir string - MaxParallel int64 - RunnerNamePrefix string - Executor string - Network string - ComputeBaseURL string - MacosBaseImageID string - MacosMachineArch string - Logger *slog.Logger + BinaryPath string + DefaultImage string + DefaultMachine string + DefaultDuration time.Duration + WorkDir string + MaxParallel int64 + RunnerNamePrefix string + Executor string + Network string + ComputeBaseURL string + MacosBaseImageID string + MacosMachineArch string + LinuxCachePath string + LinuxCacheVolumes []CacheVolume + MacosCachePath string + MacosCacheVolumes []CacheVolume + Logger *slog.Logger +} + +type CacheVolume struct { + Tag string + MountPoint string + SizeGb int64 } type LaunchRequest struct { @@ -73,6 +83,12 @@ func NewDispatcher(opts Options) (*Dispatcher, error) { if opts.DefaultDuration == 0 { opts.DefaultDuration = 30 * time.Minute } + if opts.LinuxCachePath == "" { + opts.LinuxCachePath = "/var/cache/burrow" + } + if opts.MacosCachePath == "" { + opts.MacosCachePath = "/Users/runner/.cache/burrow" + } logger := opts.Logger if logger == nil { logger = slog.New(slog.NewTextHandler(io.Discard, nil)) @@ -104,6 +120,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin } machineType := choose(req.MachineType, d.opts.DefaultMachine) image := choose(req.Image, d.opts.DefaultImage) + if req.ExtraEnv == nil { + req.ExtraEnv = make(map[string]string) + } if hasWindowsLabel(req.Labels) { if err := d.launchWindowsRunnerViaWinRM(ctx, runnerName, req, duration, machineType); err != nil { @@ -113,6 +132,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin } if hasMacOSLabel(req.Labels) { + if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok { + req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.MacosCachePath + } // Compute macOS shapes differ from the Linux "run" defaults. If the request // didn't specify a machine type, ensure we pick a macOS-valid default. if machineType == "" || machineType == d.opts.DefaultMachine { @@ -129,6 +151,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin } return runnerName, nil } + if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok { + req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.LinuxCachePath + } env := map[string]string{ "FORGEJO_INSTANCE_URL": req.InstanceURL, @@ -140,9 +165,6 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin for k, v := range req.ExtraEnv { env[k] = v } - if _, ok := env["NSC_CACHE_PATH"]; !ok { - env["NSC_CACHE_PATH"] = "/nix/store" - } script := d.bootstrapScript() args := []string{ @@ -161,6 +183,7 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin if d.opts.Network != "" { args = append(args, "--network", d.opts.Network) } + args = appendVolumeArgs(args, d.opts.LinuxCacheVolumes) for key, value := range env { if value == "" { continue @@ -370,6 +393,16 @@ func choose(values ...string) string { return "" } +func appendVolumeArgs(args []string, volumes []CacheVolume) []string { + for _, volume := range volumes { + if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 { + continue + } + args = append(args, "--volume", fmt.Sprintf("cache:%s:%s:%d", volume.Tag, volume.MountPoint, volume.SizeGb)) + } + return args +} + func (d *Dispatcher) bootstrapScript() string { var builder strings.Builder builder.WriteString(`set -euo pipefail diff --git a/services/forgejo-nsc/internal/nsc/macos.go b/services/forgejo-nsc/internal/nsc/macos.go index 9084584..e5deee7 100644 --- a/services/forgejo-nsc/internal/nsc/macos.go +++ b/services/forgejo-nsc/internal/nsc/macos.go @@ -206,12 +206,8 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r for k, v := range req.ExtraEnv { env[k] = v } - // Best-effort caching: workflows call Scripts/nscloud-cache.sh, which is a - // no-op unless NSC_CACHE_PATH is set. This may still be skipped if spacectl - // lacks credentials, but setting the path is harmless and keeps behavior - // consistent across macOS / Linux runners. if _, ok := env["NSC_CACHE_PATH"]; !ok { - env["NSC_CACHE_PATH"] = "/Users/runner/.cache/nscloud" + env["NSC_CACHE_PATH"] = d.opts.MacosCachePath } deadline := timestamppb.New(time.Now().Add(ttl)) @@ -243,10 +239,15 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r }, }, } + experimental := &computev1beta.CreateInstanceRequest_ExperimentalFeatures{} if imageID := macosComputeBaseImageID(d.opts.MacosBaseImageID); imageID != "" { - createReq.Experimental = &computev1beta.CreateInstanceRequest_ExperimentalFeatures{ - MacosBaseImageId: imageID, - } + experimental.MacosBaseImageId = imageID + } + if volumes := computeCacheVolumeRequests(d.opts.MacosCacheVolumes); len(volumes) > 0 { + experimental.Volumes = volumes + } + if experimental.MacosBaseImageId != "" || len(experimental.Volumes) > 0 { + createReq.Experimental = experimental } d.log.Info("launching Namespace macos runner", @@ -572,6 +573,22 @@ func (d *Dispatcher) destroyComputeInstance(ctx context.Context, client computev d.log.Info("macos runner destroyed", "runner", runnerName, "instance", instanceID) } +func computeCacheVolumeRequests(volumes []CacheVolume) []*computev1beta.VolumeRequest { + var out []*computev1beta.VolumeRequest + for _, volume := range volumes { + if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 { + continue + } + out = append(out, &computev1beta.VolumeRequest{ + MountPoint: volume.MountPoint, + Tag: volume.Tag, + SizeMb: volume.SizeGb * 1024, + PersistencyKind: computev1beta.VolumeRequest_CACHE, + }) + } + return out +} + func macosBootstrapScript() string { // Keep this script self-contained: it runs on a fresh macOS VM base image. var b strings.Builder diff --git a/services/forgejo-nsc/internal/nsc/macos_nsc.go b/services/forgejo-nsc/internal/nsc/macos_nsc.go index e7b8023..26cbab0 100644 --- a/services/forgejo-nsc/internal/nsc/macos_nsc.go +++ b/services/forgejo-nsc/internal/nsc/macos_nsc.go @@ -144,6 +144,7 @@ func (d *Dispatcher) launchMacOSRunnerViaNSC(ctx context.Context, runnerName str "--wait_timeout", a.waitTimeout.String(), } args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL) + args = appendVolumeArgs(args, d.opts.MacosCacheVolumes) createCtx, cancel := context.WithTimeout(ctx, a.createTimeout) defer cancel()