diff --git a/.forgejo/workflows/build-apple.yml b/.forgejo/workflows/build-apple.yml index fd69acc..e154a98 100644 --- a/.forgejo/workflows/build-apple.yml +++ b/.forgejo/workflows/build-apple.yml @@ -85,6 +85,12 @@ jobs: "${shared_root}/apple/SourcePackages" \ "${lane_root}/cargo-target" \ "${lane_root}/DerivedData" + rm -rf \ + "${lane_root}/cargo-target" \ + "${lane_root}/DerivedData" + mkdir -p \ + "${lane_root}/cargo-target" \ + "${lane_root}/DerivedData" echo "CARGO_HOME=${shared_root}/cargo" >> "${GITHUB_ENV}" echo "CARGO_TARGET_DIR=${lane_root}/cargo-target" >> "${GITHUB_ENV}" echo "RUSTUP_HOME=${shared_root}/rustup" >> "${GITHUB_ENV}" diff --git a/.forgejo/workflows/build-rust.yml b/.forgejo/workflows/build-rust.yml index 17bcea1..d70dcf0 100644 --- a/.forgejo/workflows/build-rust.yml +++ b/.forgejo/workflows/build-rust.yml @@ -16,7 +16,7 @@ concurrency: jobs: rust: name: Cargo Test - runs-on: namespace-profile-linux-medium + runs-on: [self-hosted, linux, x86_64, burrow-forge] env: CARGO_INCREMENTAL: 0 RUSTC_WRAPPER: sccache @@ -32,19 +32,11 @@ jobs: shell: bash run: | set -euo pipefail - cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}" - shared_root="${NSC_SHARED_CACHE_PATH:-${cache_root}/shared}" - lane_root="${NSC_LANE_CACHE_PATH:-${cache_root}/lane/build-rust}" - mkdir -p \ - "${shared_root}/cargo" \ - "${shared_root}/sccache" \ - "${shared_root}/xdg" \ - "${lane_root}/cargo-target" - echo "CARGO_HOME=${shared_root}/cargo" >> "${GITHUB_ENV}" - echo "SCCACHE_DIR=${shared_root}/sccache" >> "${GITHUB_ENV}" - echo "XDG_CACHE_HOME=${shared_root}/xdg" >> "${GITHUB_ENV}" - echo "CARGO_TARGET_DIR=${lane_root}/cargo-target" >> "${GITHUB_ENV}" - df -h /nix "${shared_root}" "${lane_root}" || true + cache_root="${HOME}/.cache/burrow" + mkdir -p "${cache_root}/cargo" "${cache_root}/sccache" "${cache_root}/cargo-target/build-rust" + echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}" + echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}" + echo "CARGO_TARGET_DIR=${cache_root}/cargo-target/build-rust" >> "${GITHUB_ENV}" - name: Test shell: bash diff --git a/.forgejo/workflows/build-site.yml b/.forgejo/workflows/build-site.yml index 9b08152..de296d4 100644 --- a/.forgejo/workflows/build-site.yml +++ b/.forgejo/workflows/build-site.yml @@ -16,7 +16,7 @@ concurrency: jobs: site: name: Next.js Build - runs-on: namespace-profile-linux-medium + runs-on: [self-hosted, linux, x86_64, burrow-forge] steps: - name: Checkout uses: https://code.forgejo.org/actions/checkout@v4 @@ -28,27 +28,12 @@ jobs: shell: bash run: | set -euo pipefail - cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}" - shared_root="${NSC_SHARED_CACHE_PATH:-${cache_root}/shared}" - lane_root="${NSC_LANE_CACHE_PATH:-${cache_root}/lane/build-site}" - mkdir -p \ - "${shared_root}/npm" \ - "${shared_root}/xdg" \ - "${lane_root}/next-cache" - echo "NPM_CONFIG_CACHE=${shared_root}/npm" >> "${GITHUB_ENV}" - echo "XDG_CACHE_HOME=${shared_root}/xdg" >> "${GITHUB_ENV}" - echo "NEXT_CACHE_DIR=${lane_root}/next-cache" >> "${GITHUB_ENV}" - df -h /nix "${shared_root}" "${lane_root}" || true + cache_root="${HOME}/.cache/burrow" + mkdir -p "${cache_root}/npm" + echo "NPM_CONFIG_CACHE=${cache_root}/npm" >> "${GITHUB_ENV}" - name: Build shell: bash run: | set -euo pipefail - nix develop .#ci -c bash -lc ' - mkdir -p site/.next - rm -rf site/.next/cache - ln -sfn "${NEXT_CACHE_DIR}" site/.next/cache - cd site - npm install - npm run build - ' + nix develop .#ci -c bash -lc 'cd site && npm install && npm run build' diff --git a/Apple/NetworkExtension/PacketTunnelProvider.swift b/Apple/NetworkExtension/PacketTunnelProvider.swift index 54b813c..98bf841 100644 --- a/Apple/NetworkExtension/PacketTunnelProvider.swift +++ b/Apple/NetworkExtension/PacketTunnelProvider.swift @@ -5,17 +5,19 @@ import libburrow @preconcurrency import NetworkExtension import os -// Xcode 26 imports `startTunnel(options:)` as `[String: NSObject]?` and treats the -// override as crossing a nonisolated boundary. The extension target does not -// mutate or forward these Cocoa objects, so treat them as an unchecked escape hatch. -extension NSObject: @retroactive @unchecked Sendable {} - class PacketTunnelProvider: NEPacketTunnelProvider { enum Error: Swift.Error { case missingTunnelConfiguration } - private static let logger = Logger.logger(for: PacketTunnelProvider.self) + private let logger = Logger.logger(for: PacketTunnelProvider.self) + + private var client: TunnelClient { + get throws { try _client.get() } + } + private let _client: Result = Result { + try TunnelClient.unix(socketURL: Constants.socketURL) + } override init() { do { @@ -24,33 +26,31 @@ class PacketTunnelProvider: NEPacketTunnelProvider { databasePath: try Constants.databaseURL.path(percentEncoded: false) ) } catch { - Self.logger.error("Failed to spawn networking thread: \(error)") + logger.error("Failed to spawn networking thread: \(error)") } } - nonisolated override func startTunnel(options: [String: NSObject]? = nil) async throws { + override func startTunnel(options: [String: NSObject]? = nil) async throws { do { - let client = try TunnelClient.unix(socketURL: Constants.socketURL) let configuration = try await Array(client.tunnelConfiguration(.init()).prefix(1)).first guard let settings = configuration?.settings else { throw Error.missingTunnelConfiguration } try await setTunnelNetworkSettings(settings) _ = try await client.tunnelStart(.init()) - Self.logger.log("Started tunnel with network settings: \(settings)") + logger.log("Started tunnel with network settings: \(settings)") } catch { - Self.logger.error("Failed to start tunnel: \(error)") + logger.error("Failed to start tunnel: \(error)") throw error } } - nonisolated override func stopTunnel(with reason: NEProviderStopReason) async { + override func stopTunnel(with reason: NEProviderStopReason) async { do { - let client = try TunnelClient.unix(socketURL: Constants.socketURL) _ = try await client.tunnelStop(.init()) - Self.logger.log("Stopped client") + logger.log("Stopped client") } catch { - Self.logger.error("Failed to stop tunnel: \(error)") + logger.error("Failed to stop tunnel: \(error)") } } } diff --git a/Scripts/_burrow-secrets.sh b/Scripts/_burrow-secrets.sh index e08bf2a..7754b74 100644 --- a/Scripts/_burrow-secrets.sh +++ b/Scripts/_burrow-secrets.sh @@ -84,13 +84,13 @@ burrow_resolve_secret_file() { return 0 fi - if [[ -n "${age_path}" && -f "${age_path}" ]]; then - burrow_decrypt_age_secret_to_temp "${repo_root}" "${age_path}" + if [[ -n "${intake_path}" && -s "${intake_path}" ]]; then + printf '%s\n' "${intake_path}" return 0 fi - if [[ -n "${intake_path}" && -s "${intake_path}" ]]; then - printf '%s\n' "${intake_path}" + if [[ -n "${age_path}" && -f "${age_path}" ]]; then + burrow_decrypt_age_secret_to_temp "${repo_root}" "${age_path}" return 0 fi diff --git a/Scripts/provision-forgejo-nsc.sh b/Scripts/provision-forgejo-nsc.sh index b8c9f12..c85b993 100755 --- a/Scripts/provision-forgejo-nsc.sh +++ b/Scripts/provision-forgejo-nsc.sh @@ -28,6 +28,7 @@ Options: --contact-user Forgejo username used for PAT creation (default: contact) --scope-owner Forgejo org/user owner for the default NSC scope (default: hackclub) --scope-name Forgejo repository name for the default NSC scope (default: burrow) + --write-intake Also write plaintext runtime inputs to intake/ for local debugging. -h, --help Show this help text. EOF } @@ -42,6 +43,7 @@ CONTACT_USER="${FORGEJO_CONTACT_USER:-contact}" SCOPE_OWNER="${FORGEJO_SCOPE_OWNER:-hackclub}" SCOPE_NAME="${FORGEJO_SCOPE_NAME:-burrow}" BURROW_FLAKE_TMPDIRS=() +WRITE_INTAKE=0 TMP_DIR="" cleanup() { @@ -85,6 +87,10 @@ while [[ $# -gt 0 ]]; do SCOPE_NAME="${2:?missing value for --scope-name}" shift 2 ;; + --write-intake) + WRITE_INTAKE=1 + shift + ;; -h|--help) usage exit 0 @@ -168,6 +174,8 @@ PY chmod 600 "${token_file}" elif [[ -f "${token_secret}" ]]; then burrow_decrypt_age_secret_to_temp "${REPO_ROOT}" "${token_secret}" > "${token_file}" +elif [[ -s "${REPO_ROOT}/intake/forgejo_nsc_token.txt" ]]; then + cp "${REPO_ROOT}/intake/forgejo_nsc_token.txt" "${token_file}" fi if [[ -s "${token_file}" ]]; then @@ -290,5 +298,20 @@ burrow_encrypt_secret_from_file "${REPO_ROOT}" "${token_secret}" "${token_file}" burrow_encrypt_secret_from_file "${REPO_ROOT}" "${dispatcher_secret}" "${dispatcher_out}" burrow_encrypt_secret_from_file "${REPO_ROOT}" "${autoscaler_secret}" "${autoscaler_out}" +if [[ "${WRITE_INTAKE}" -eq 1 ]]; then + mkdir -p "${REPO_ROOT}/intake" + chmod 700 "${REPO_ROOT}/intake" + cp "${token_file}" "${REPO_ROOT}/intake/forgejo_nsc_token.txt" + cp "${dispatcher_out}" "${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml" + cp "${autoscaler_out}" "${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml" + chmod 600 \ + "${REPO_ROOT}/intake/forgejo_nsc_token.txt" \ + "${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml" \ + "${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml" +fi + echo "Updated secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age." +if [[ "${WRITE_INTAKE}" -eq 1 ]]; then + echo "Also refreshed intake/forgejo_nsc_{token,dispatcher,autoscaler} for local debugging." +fi echo "Minted Forgejo PAT ${token_name} for ${CONTACT_USER} on ${HOST}." diff --git a/Scripts/sync-forgejo-nsc-config.sh b/Scripts/sync-forgejo-nsc-config.sh index 431f832..baa4960 100755 --- a/Scripts/sync-forgejo-nsc-config.sh +++ b/Scripts/sync-forgejo-nsc-config.sh @@ -5,13 +5,14 @@ usage() { cat <<'EOF' Usage: Scripts/sync-forgejo-nsc-config.sh [options] -Deploy Burrow forgejo-nsc runtime inputs from age secrets onto the forge host. +Copy Burrow forgejo-nsc runtime inputs from age secrets or intake/ onto the forge host and +restart the dispatcher/autoscaler units. Options: --host SSH target (default: root@git.burrow.net) --ssh-key SSH private key (default: secrets/forgejo/agent-ssh-key.age, then intake/) - --rotate-pat Re-render the encrypted runtime inputs before deploying. - --no-restart Validate the encrypted inputs only; do not deploy. + --rotate-pat Re-render the intake files before syncing. + --no-restart Copy files only. -h, --help Show this help text. EOF } @@ -74,6 +75,7 @@ burrow_require_cmd() { } burrow_require_cmd ssh +burrow_require_cmd scp SSH_KEY="$( burrow_resolve_secret_file \ @@ -88,25 +90,26 @@ if [[ "${ROTATE_PAT}" -eq 1 ]]; then "${SCRIPT_DIR}/provision-forgejo-nsc.sh" --host "${HOST}" --ssh-key "${SSH_KEY}" fi +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/burrow-nsc-sync.XXXXXX")" token_file="$( burrow_resolve_secret_file \ "${REPO_ROOT}" \ "" \ - "" \ + "${REPO_ROOT}/intake/forgejo_nsc_token.txt" \ "${REPO_ROOT}/secrets/forgejo/nsc-token.age" )" dispatcher_file="$( burrow_resolve_secret_file \ "${REPO_ROOT}" \ "" \ - "" \ + "${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml" \ "${REPO_ROOT}/secrets/forgejo/nsc-dispatcher-config.age" )" autoscaler_file="$( burrow_resolve_secret_file \ "${REPO_ROOT}" \ "" \ - "" \ + "${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml" \ "${REPO_ROOT}/secrets/forgejo/nsc-autoscaler-config.age" )" @@ -117,11 +120,45 @@ for path in "${token_file}" "${dispatcher_file}" "${autoscaler_file}"; do fi done +ssh_opts=( + -i "${SSH_KEY}" + -o IdentitiesOnly=yes + -o UserKnownHostsFile="${KNOWN_HOSTS_FILE}" + -o StrictHostKeyChecking=accept-new +) + +remote_tmp="$(ssh "${ssh_opts[@]}" "${HOST}" "mktemp -d")" +cleanup_remote() { + if [[ -n "${remote_tmp:-}" ]]; then + ssh "${ssh_opts[@]}" "${HOST}" "rm -rf '${remote_tmp}'" >/dev/null 2>&1 || true + fi +} +trap 'cleanup_remote; cleanup' EXIT + +scp "${ssh_opts[@]}" \ + "${token_file}" \ + "${dispatcher_file}" \ + "${autoscaler_file}" \ + "${HOST}:${remote_tmp}/" + +ssh "${ssh_opts[@]}" "${HOST}" " + set -euo pipefail + install -d -m 0755 /var/lib/burrow/intake + install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${token_file}")' /var/lib/burrow/intake/forgejo_nsc_token.txt + install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${dispatcher_file}")' /var/lib/burrow/intake/forgejo_nsc_dispatcher.yaml + install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${autoscaler_file}")' /var/lib/burrow/intake/forgejo_nsc_autoscaler.yaml +" + if [[ "${NO_RESTART}" -eq 0 ]]; then - BURROW_FORGE_HOST="${HOST}" \ - BURROW_FORGE_SSH_KEY="${SSH_KEY}" \ - BURROW_FORGE_KNOWN_HOSTS_FILE="${KNOWN_HOSTS_FILE}" \ - "${SCRIPT_DIR}/forge-deploy.sh" --switch + ssh "${ssh_opts[@]}" "${HOST}" " + set -euo pipefail + systemctl restart forgejo-nsc-dispatcher.service forgejo-nsc-autoscaler.service + systemctl is-active forgejo-nsc-dispatcher.service forgejo-nsc-autoscaler.service + ls -l \ + /var/lib/burrow/intake/forgejo_nsc_token.txt \ + /var/lib/burrow/intake/forgejo_nsc_dispatcher.yaml \ + /var/lib/burrow/intake/forgejo_nsc_autoscaler.yaml + " fi -echo "forgejo-nsc runtime sync complete (host=${HOST}, deployed=$((1 - NO_RESTART)))." +echo "forgejo-nsc runtime sync complete (host=${HOST}, restarted=$((1 - NO_RESTART)))." diff --git a/services/forgejo-nsc/README.md b/services/forgejo-nsc/README.md index 3b819d4..f928973 100644 --- a/services/forgejo-nsc/README.md +++ b/services/forgejo-nsc/README.md @@ -46,9 +46,8 @@ profile. The important knobs are: Namespace environment. The dispatcher destroys the instance after a job so the TTL acts as a hard cap, not an idle timeout. - `namespace.linux_cache_*` / `namespace.macos_cache_*` – persistent cache - volumes mounted into runners so Linux can keep `/nix` plus shared build - caches warm and macOS can reuse Rust toolchains, Xcode package caches, and - lane-local derived data. + volumes mounted into runners so Linux can keep `/nix` plus build caches warm + and macOS can reuse Rust toolchains, Xcode package caches, and derived data. ### Running locally @@ -160,8 +159,8 @@ generate a Namespace token from the logged-in Namespace account, and refresh `secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age`. The token file is emitted as JSON with a `bearer_token` field so both the Compute API path and the `nsc` CLI fallback can consume the same secret -material. The forge host consumes the encrypted secrets through agenix; avoid -keeping local plaintext `intake/` copies around. +material. Use `--write-intake` only when you explicitly need local plaintext +debug copies. Long-lived runtime state is now sourced from age-encrypted files: diff --git a/services/forgejo-nsc/config.example.yaml b/services/forgejo-nsc/config.example.yaml index 15fe0a4..fcd56ec 100644 --- a/services/forgejo-nsc/config.example.yaml +++ b/services/forgejo-nsc/config.example.yaml @@ -11,10 +11,10 @@ forgejo: timeout: "30s" namespace: - nsc_binary: "nsc" + nsc_binary: "/app/bin/nsc" compute_base_url: "https://ord4.compute.namespaceapis.com" - image: "code.forgejo.org/forgejo/runner:11" - machine_type: "4x8" + image: "ghcr.io/forgejo/runner:3" + machine_type: "8x16" macos_base_image_id: "tahoe" macos_machine_arch: "arm64" duration: "30m" @@ -31,15 +31,9 @@ namespace: size_gb: 40 macos_cache_path: "/Users/runner/.cache/burrow" macos_cache_volumes: - - tag: "burrow-forgejo-macos-shared-v1" - mount_point: "/Users/runner/.cache/burrow/shared" - size_gb: 80 - - tag: "burrow-forgejo-macos-macos-v1" - mount_point: "/Users/runner/.cache/burrow/lane/macos" - size_gb: 80 - - tag: "burrow-forgejo-macos-ios-simulator-v1" - mount_point: "/Users/runner/.cache/burrow/lane/ios-simulator" - size_gb: 80 + - tag: "burrow-forgejo-macos-cache" + mount_point: "/Users/runner/.cache/burrow" + size_gb: 60 runner: name_prefix: "nscloud-"