Compare commits

..

2 commits

Author SHA1 Message Date
Conrad Kramer
5c0a9b3f54 Work around Xcode 26 tunnel isolation
Some checks are pending
Build Apple / Build App (iOS Simulator) (push) Waiting to run
Build Apple / Build App (macOS) (push) Waiting to run
Build Rust / Cargo Test (push) Waiting to run
Build Site / Next.js Build (push) Waiting to run
2026-03-19 03:51:56 -07:00
Conrad Kramer
028627bfcb Wire namespace caches and agenix secrets 2026-03-19 03:51:53 -07:00
9 changed files with 81 additions and 117 deletions

View file

@ -85,12 +85,6 @@ jobs:
"${shared_root}/apple/SourcePackages" \ "${shared_root}/apple/SourcePackages" \
"${lane_root}/cargo-target" \ "${lane_root}/cargo-target" \
"${lane_root}/DerivedData" "${lane_root}/DerivedData"
rm -rf \
"${lane_root}/cargo-target" \
"${lane_root}/DerivedData"
mkdir -p \
"${lane_root}/cargo-target" \
"${lane_root}/DerivedData"
echo "CARGO_HOME=${shared_root}/cargo" >> "${GITHUB_ENV}" echo "CARGO_HOME=${shared_root}/cargo" >> "${GITHUB_ENV}"
echo "CARGO_TARGET_DIR=${lane_root}/cargo-target" >> "${GITHUB_ENV}" echo "CARGO_TARGET_DIR=${lane_root}/cargo-target" >> "${GITHUB_ENV}"
echo "RUSTUP_HOME=${shared_root}/rustup" >> "${GITHUB_ENV}" echo "RUSTUP_HOME=${shared_root}/rustup" >> "${GITHUB_ENV}"

View file

@ -16,7 +16,7 @@ concurrency:
jobs: jobs:
rust: rust:
name: Cargo Test name: Cargo Test
runs-on: [self-hosted, linux, x86_64, burrow-forge] runs-on: namespace-profile-linux-medium
env: env:
CARGO_INCREMENTAL: 0 CARGO_INCREMENTAL: 0
RUSTC_WRAPPER: sccache RUSTC_WRAPPER: sccache
@ -32,11 +32,19 @@ jobs:
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
cache_root="${HOME}/.cache/burrow" cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}"
mkdir -p "${cache_root}/cargo" "${cache_root}/sccache" "${cache_root}/cargo-target/build-rust" shared_root="${NSC_SHARED_CACHE_PATH:-${cache_root}/shared}"
echo "CARGO_HOME=${cache_root}/cargo" >> "${GITHUB_ENV}" lane_root="${NSC_LANE_CACHE_PATH:-${cache_root}/lane/build-rust}"
echo "SCCACHE_DIR=${cache_root}/sccache" >> "${GITHUB_ENV}" mkdir -p \
echo "CARGO_TARGET_DIR=${cache_root}/cargo-target/build-rust" >> "${GITHUB_ENV}" "${shared_root}/cargo" \
"${shared_root}/sccache" \
"${shared_root}/xdg" \
"${lane_root}/cargo-target"
echo "CARGO_HOME=${shared_root}/cargo" >> "${GITHUB_ENV}"
echo "SCCACHE_DIR=${shared_root}/sccache" >> "${GITHUB_ENV}"
echo "XDG_CACHE_HOME=${shared_root}/xdg" >> "${GITHUB_ENV}"
echo "CARGO_TARGET_DIR=${lane_root}/cargo-target" >> "${GITHUB_ENV}"
df -h /nix "${shared_root}" "${lane_root}" || true
- name: Test - name: Test
shell: bash shell: bash

View file

@ -16,7 +16,7 @@ concurrency:
jobs: jobs:
site: site:
name: Next.js Build name: Next.js Build
runs-on: [self-hosted, linux, x86_64, burrow-forge] runs-on: namespace-profile-linux-medium
steps: steps:
- name: Checkout - name: Checkout
uses: https://code.forgejo.org/actions/checkout@v4 uses: https://code.forgejo.org/actions/checkout@v4
@ -28,12 +28,27 @@ jobs:
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
cache_root="${HOME}/.cache/burrow" cache_root="${NSC_CACHE_PATH:-${HOME}/.cache/burrow}"
mkdir -p "${cache_root}/npm" shared_root="${NSC_SHARED_CACHE_PATH:-${cache_root}/shared}"
echo "NPM_CONFIG_CACHE=${cache_root}/npm" >> "${GITHUB_ENV}" lane_root="${NSC_LANE_CACHE_PATH:-${cache_root}/lane/build-site}"
mkdir -p \
"${shared_root}/npm" \
"${shared_root}/xdg" \
"${lane_root}/next-cache"
echo "NPM_CONFIG_CACHE=${shared_root}/npm" >> "${GITHUB_ENV}"
echo "XDG_CACHE_HOME=${shared_root}/xdg" >> "${GITHUB_ENV}"
echo "NEXT_CACHE_DIR=${lane_root}/next-cache" >> "${GITHUB_ENV}"
df -h /nix "${shared_root}" "${lane_root}" || true
- name: Build - name: Build
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
nix develop .#ci -c bash -lc 'cd site && npm install && npm run build' nix develop .#ci -c bash -lc '
mkdir -p site/.next
rm -rf site/.next/cache
ln -sfn "${NEXT_CACHE_DIR}" site/.next/cache
cd site
npm install
npm run build
'

View file

@ -5,19 +5,17 @@ import libburrow
@preconcurrency import NetworkExtension @preconcurrency import NetworkExtension
import os import os
// Xcode 26 imports `startTunnel(options:)` as `[String: NSObject]?` and treats the
// override as crossing a nonisolated boundary. The extension target does not
// mutate or forward these Cocoa objects, so treat them as an unchecked escape hatch.
extension NSObject: @retroactive @unchecked Sendable {}
class PacketTunnelProvider: NEPacketTunnelProvider { class PacketTunnelProvider: NEPacketTunnelProvider {
enum Error: Swift.Error { enum Error: Swift.Error {
case missingTunnelConfiguration case missingTunnelConfiguration
} }
private let logger = Logger.logger(for: PacketTunnelProvider.self) private static let logger = Logger.logger(for: PacketTunnelProvider.self)
private var client: TunnelClient {
get throws { try _client.get() }
}
private let _client: Result<TunnelClient, Swift.Error> = Result {
try TunnelClient.unix(socketURL: Constants.socketURL)
}
override init() { override init() {
do { do {
@ -26,31 +24,33 @@ class PacketTunnelProvider: NEPacketTunnelProvider {
databasePath: try Constants.databaseURL.path(percentEncoded: false) databasePath: try Constants.databaseURL.path(percentEncoded: false)
) )
} catch { } catch {
logger.error("Failed to spawn networking thread: \(error)") Self.logger.error("Failed to spawn networking thread: \(error)")
} }
} }
override func startTunnel(options: [String: NSObject]? = nil) async throws { nonisolated override func startTunnel(options: [String: NSObject]? = nil) async throws {
do { do {
let client = try TunnelClient.unix(socketURL: Constants.socketURL)
let configuration = try await Array(client.tunnelConfiguration(.init()).prefix(1)).first let configuration = try await Array(client.tunnelConfiguration(.init()).prefix(1)).first
guard let settings = configuration?.settings else { guard let settings = configuration?.settings else {
throw Error.missingTunnelConfiguration throw Error.missingTunnelConfiguration
} }
try await setTunnelNetworkSettings(settings) try await setTunnelNetworkSettings(settings)
_ = try await client.tunnelStart(.init()) _ = try await client.tunnelStart(.init())
logger.log("Started tunnel with network settings: \(settings)") Self.logger.log("Started tunnel with network settings: \(settings)")
} catch { } catch {
logger.error("Failed to start tunnel: \(error)") Self.logger.error("Failed to start tunnel: \(error)")
throw error throw error
} }
} }
override func stopTunnel(with reason: NEProviderStopReason) async { nonisolated override func stopTunnel(with reason: NEProviderStopReason) async {
do { do {
let client = try TunnelClient.unix(socketURL: Constants.socketURL)
_ = try await client.tunnelStop(.init()) _ = try await client.tunnelStop(.init())
logger.log("Stopped client") Self.logger.log("Stopped client")
} catch { } catch {
logger.error("Failed to stop tunnel: \(error)") Self.logger.error("Failed to stop tunnel: \(error)")
} }
} }
} }

View file

@ -84,13 +84,13 @@ burrow_resolve_secret_file() {
return 0 return 0
fi fi
if [[ -n "${intake_path}" && -s "${intake_path}" ]]; then if [[ -n "${age_path}" && -f "${age_path}" ]]; then
printf '%s\n' "${intake_path}" burrow_decrypt_age_secret_to_temp "${repo_root}" "${age_path}"
return 0 return 0
fi fi
if [[ -n "${age_path}" && -f "${age_path}" ]]; then if [[ -n "${intake_path}" && -s "${intake_path}" ]]; then
burrow_decrypt_age_secret_to_temp "${repo_root}" "${age_path}" printf '%s\n' "${intake_path}"
return 0 return 0
fi fi

View file

@ -28,7 +28,6 @@ Options:
--contact-user <name> Forgejo username used for PAT creation (default: contact) --contact-user <name> Forgejo username used for PAT creation (default: contact)
--scope-owner <name> Forgejo org/user owner for the default NSC scope (default: hackclub) --scope-owner <name> Forgejo org/user owner for the default NSC scope (default: hackclub)
--scope-name <name> Forgejo repository name for the default NSC scope (default: burrow) --scope-name <name> Forgejo repository name for the default NSC scope (default: burrow)
--write-intake Also write plaintext runtime inputs to intake/ for local debugging.
-h, --help Show this help text. -h, --help Show this help text.
EOF EOF
} }
@ -43,7 +42,6 @@ CONTACT_USER="${FORGEJO_CONTACT_USER:-contact}"
SCOPE_OWNER="${FORGEJO_SCOPE_OWNER:-hackclub}" SCOPE_OWNER="${FORGEJO_SCOPE_OWNER:-hackclub}"
SCOPE_NAME="${FORGEJO_SCOPE_NAME:-burrow}" SCOPE_NAME="${FORGEJO_SCOPE_NAME:-burrow}"
BURROW_FLAKE_TMPDIRS=() BURROW_FLAKE_TMPDIRS=()
WRITE_INTAKE=0
TMP_DIR="" TMP_DIR=""
cleanup() { cleanup() {
@ -87,10 +85,6 @@ while [[ $# -gt 0 ]]; do
SCOPE_NAME="${2:?missing value for --scope-name}" SCOPE_NAME="${2:?missing value for --scope-name}"
shift 2 shift 2
;; ;;
--write-intake)
WRITE_INTAKE=1
shift
;;
-h|--help) -h|--help)
usage usage
exit 0 exit 0
@ -174,8 +168,6 @@ PY
chmod 600 "${token_file}" chmod 600 "${token_file}"
elif [[ -f "${token_secret}" ]]; then elif [[ -f "${token_secret}" ]]; then
burrow_decrypt_age_secret_to_temp "${REPO_ROOT}" "${token_secret}" > "${token_file}" burrow_decrypt_age_secret_to_temp "${REPO_ROOT}" "${token_secret}" > "${token_file}"
elif [[ -s "${REPO_ROOT}/intake/forgejo_nsc_token.txt" ]]; then
cp "${REPO_ROOT}/intake/forgejo_nsc_token.txt" "${token_file}"
fi fi
if [[ -s "${token_file}" ]]; then if [[ -s "${token_file}" ]]; then
@ -298,20 +290,5 @@ burrow_encrypt_secret_from_file "${REPO_ROOT}" "${token_secret}" "${token_file}"
burrow_encrypt_secret_from_file "${REPO_ROOT}" "${dispatcher_secret}" "${dispatcher_out}" burrow_encrypt_secret_from_file "${REPO_ROOT}" "${dispatcher_secret}" "${dispatcher_out}"
burrow_encrypt_secret_from_file "${REPO_ROOT}" "${autoscaler_secret}" "${autoscaler_out}" burrow_encrypt_secret_from_file "${REPO_ROOT}" "${autoscaler_secret}" "${autoscaler_out}"
if [[ "${WRITE_INTAKE}" -eq 1 ]]; then
mkdir -p "${REPO_ROOT}/intake"
chmod 700 "${REPO_ROOT}/intake"
cp "${token_file}" "${REPO_ROOT}/intake/forgejo_nsc_token.txt"
cp "${dispatcher_out}" "${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml"
cp "${autoscaler_out}" "${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml"
chmod 600 \
"${REPO_ROOT}/intake/forgejo_nsc_token.txt" \
"${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml" \
"${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml"
fi
echo "Updated secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age." echo "Updated secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age."
if [[ "${WRITE_INTAKE}" -eq 1 ]]; then
echo "Also refreshed intake/forgejo_nsc_{token,dispatcher,autoscaler} for local debugging."
fi
echo "Minted Forgejo PAT ${token_name} for ${CONTACT_USER} on ${HOST}." echo "Minted Forgejo PAT ${token_name} for ${CONTACT_USER} on ${HOST}."

View file

@ -5,14 +5,13 @@ usage() {
cat <<'EOF' cat <<'EOF'
Usage: Scripts/sync-forgejo-nsc-config.sh [options] Usage: Scripts/sync-forgejo-nsc-config.sh [options]
Copy Burrow forgejo-nsc runtime inputs from age secrets or intake/ onto the forge host and Deploy Burrow forgejo-nsc runtime inputs from age secrets onto the forge host.
restart the dispatcher/autoscaler units.
Options: Options:
--host <user@host> SSH target (default: root@git.burrow.net) --host <user@host> SSH target (default: root@git.burrow.net)
--ssh-key <path> SSH private key (default: secrets/forgejo/agent-ssh-key.age, then intake/) --ssh-key <path> SSH private key (default: secrets/forgejo/agent-ssh-key.age, then intake/)
--rotate-pat Re-render the intake files before syncing. --rotate-pat Re-render the encrypted runtime inputs before deploying.
--no-restart Copy files only. --no-restart Validate the encrypted inputs only; do not deploy.
-h, --help Show this help text. -h, --help Show this help text.
EOF EOF
} }
@ -75,7 +74,6 @@ burrow_require_cmd() {
} }
burrow_require_cmd ssh burrow_require_cmd ssh
burrow_require_cmd scp
SSH_KEY="$( SSH_KEY="$(
burrow_resolve_secret_file \ burrow_resolve_secret_file \
@ -90,26 +88,25 @@ if [[ "${ROTATE_PAT}" -eq 1 ]]; then
"${SCRIPT_DIR}/provision-forgejo-nsc.sh" --host "${HOST}" --ssh-key "${SSH_KEY}" "${SCRIPT_DIR}/provision-forgejo-nsc.sh" --host "${HOST}" --ssh-key "${SSH_KEY}"
fi fi
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/burrow-nsc-sync.XXXXXX")"
token_file="$( token_file="$(
burrow_resolve_secret_file \ burrow_resolve_secret_file \
"${REPO_ROOT}" \ "${REPO_ROOT}" \
"" \ "" \
"${REPO_ROOT}/intake/forgejo_nsc_token.txt" \ "" \
"${REPO_ROOT}/secrets/forgejo/nsc-token.age" "${REPO_ROOT}/secrets/forgejo/nsc-token.age"
)" )"
dispatcher_file="$( dispatcher_file="$(
burrow_resolve_secret_file \ burrow_resolve_secret_file \
"${REPO_ROOT}" \ "${REPO_ROOT}" \
"" \ "" \
"${REPO_ROOT}/intake/forgejo_nsc_dispatcher.yaml" \ "" \
"${REPO_ROOT}/secrets/forgejo/nsc-dispatcher-config.age" "${REPO_ROOT}/secrets/forgejo/nsc-dispatcher-config.age"
)" )"
autoscaler_file="$( autoscaler_file="$(
burrow_resolve_secret_file \ burrow_resolve_secret_file \
"${REPO_ROOT}" \ "${REPO_ROOT}" \
"" \ "" \
"${REPO_ROOT}/intake/forgejo_nsc_autoscaler.yaml" \ "" \
"${REPO_ROOT}/secrets/forgejo/nsc-autoscaler-config.age" "${REPO_ROOT}/secrets/forgejo/nsc-autoscaler-config.age"
)" )"
@ -120,45 +117,11 @@ for path in "${token_file}" "${dispatcher_file}" "${autoscaler_file}"; do
fi fi
done done
ssh_opts=(
-i "${SSH_KEY}"
-o IdentitiesOnly=yes
-o UserKnownHostsFile="${KNOWN_HOSTS_FILE}"
-o StrictHostKeyChecking=accept-new
)
remote_tmp="$(ssh "${ssh_opts[@]}" "${HOST}" "mktemp -d")"
cleanup_remote() {
if [[ -n "${remote_tmp:-}" ]]; then
ssh "${ssh_opts[@]}" "${HOST}" "rm -rf '${remote_tmp}'" >/dev/null 2>&1 || true
fi
}
trap 'cleanup_remote; cleanup' EXIT
scp "${ssh_opts[@]}" \
"${token_file}" \
"${dispatcher_file}" \
"${autoscaler_file}" \
"${HOST}:${remote_tmp}/"
ssh "${ssh_opts[@]}" "${HOST}" "
set -euo pipefail
install -d -m 0755 /var/lib/burrow/intake
install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${token_file}")' /var/lib/burrow/intake/forgejo_nsc_token.txt
install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${dispatcher_file}")' /var/lib/burrow/intake/forgejo_nsc_dispatcher.yaml
install -m 0400 -o forgejo-nsc -g forgejo-nsc '${remote_tmp}/$(basename "${autoscaler_file}")' /var/lib/burrow/intake/forgejo_nsc_autoscaler.yaml
"
if [[ "${NO_RESTART}" -eq 0 ]]; then if [[ "${NO_RESTART}" -eq 0 ]]; then
ssh "${ssh_opts[@]}" "${HOST}" " BURROW_FORGE_HOST="${HOST}" \
set -euo pipefail BURROW_FORGE_SSH_KEY="${SSH_KEY}" \
systemctl restart forgejo-nsc-dispatcher.service forgejo-nsc-autoscaler.service BURROW_FORGE_KNOWN_HOSTS_FILE="${KNOWN_HOSTS_FILE}" \
systemctl is-active forgejo-nsc-dispatcher.service forgejo-nsc-autoscaler.service "${SCRIPT_DIR}/forge-deploy.sh" --switch
ls -l \
/var/lib/burrow/intake/forgejo_nsc_token.txt \
/var/lib/burrow/intake/forgejo_nsc_dispatcher.yaml \
/var/lib/burrow/intake/forgejo_nsc_autoscaler.yaml
"
fi fi
echo "forgejo-nsc runtime sync complete (host=${HOST}, restarted=$((1 - NO_RESTART)))." echo "forgejo-nsc runtime sync complete (host=${HOST}, deployed=$((1 - NO_RESTART)))."

View file

@ -46,8 +46,9 @@ profile. The important knobs are:
Namespace environment. The dispatcher destroys the instance after a job so the Namespace environment. The dispatcher destroys the instance after a job so the
TTL acts as a hard cap, not an idle timeout. TTL acts as a hard cap, not an idle timeout.
- `namespace.linux_cache_*` / `namespace.macos_cache_*` persistent cache - `namespace.linux_cache_*` / `namespace.macos_cache_*` persistent cache
volumes mounted into runners so Linux can keep `/nix` plus build caches warm volumes mounted into runners so Linux can keep `/nix` plus shared build
and macOS can reuse Rust toolchains, Xcode package caches, and derived data. caches warm and macOS can reuse Rust toolchains, Xcode package caches, and
lane-local derived data.
### Running locally ### Running locally
@ -159,8 +160,8 @@ generate a Namespace token from the logged-in Namespace account, and refresh
`secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age`. `secrets/forgejo/{nsc-token,nsc-dispatcher-config,nsc-autoscaler-config}.age`.
The token file is emitted as JSON with a `bearer_token` field so both the The token file is emitted as JSON with a `bearer_token` field so both the
Compute API path and the `nsc` CLI fallback can consume the same secret Compute API path and the `nsc` CLI fallback can consume the same secret
material. Use `--write-intake` only when you explicitly need local plaintext material. The forge host consumes the encrypted secrets through agenix; avoid
debug copies. keeping local plaintext `intake/` copies around.
Long-lived runtime state is now sourced from age-encrypted files: Long-lived runtime state is now sourced from age-encrypted files:

View file

@ -11,10 +11,10 @@ forgejo:
timeout: "30s" timeout: "30s"
namespace: namespace:
nsc_binary: "/app/bin/nsc" nsc_binary: "nsc"
compute_base_url: "https://ord4.compute.namespaceapis.com" compute_base_url: "https://ord4.compute.namespaceapis.com"
image: "ghcr.io/forgejo/runner:3" image: "code.forgejo.org/forgejo/runner:11"
machine_type: "8x16" machine_type: "4x8"
macos_base_image_id: "tahoe" macos_base_image_id: "tahoe"
macos_machine_arch: "arm64" macos_machine_arch: "arm64"
duration: "30m" duration: "30m"
@ -31,9 +31,15 @@ namespace:
size_gb: 40 size_gb: 40
macos_cache_path: "/Users/runner/.cache/burrow" macos_cache_path: "/Users/runner/.cache/burrow"
macos_cache_volumes: macos_cache_volumes:
- tag: "burrow-forgejo-macos-cache" - tag: "burrow-forgejo-macos-shared-v1"
mount_point: "/Users/runner/.cache/burrow" mount_point: "/Users/runner/.cache/burrow/shared"
size_gb: 60 size_gb: 80
- tag: "burrow-forgejo-macos-macos-v1"
mount_point: "/Users/runner/.cache/burrow/lane/macos"
size_gb: 80
- tag: "burrow-forgejo-macos-ios-simulator-v1"
mount_point: "/Users/runner/.cache/burrow/lane/ios-simulator"
size_gb: 80
runner: runner:
name_prefix: "nscloud-" name_prefix: "nscloud-"