Wire runner caches and forge secrets through agenix

services/forgejo-nsc/README.md

@@ -45,6 +45,9 @@ profile. The important knobs are:
 - `namespace.machine_type` / `namespace.duration` – shape + TTL for the ephemeral
   Namespace environment. The dispatcher destroys the instance after a job so the
   TTL acts as a hard cap, not an idle timeout.
+- `namespace.linux_cache_*` / `namespace.macos_cache_*` – persistent cache
+  volumes mounted into runners so Linux can keep `/nix` plus build caches warm
+  and macOS can reuse Rust toolchains, Xcode package caches, and derived data.
 
 ### Running locally
 
@@ -160,12 +163,15 @@ consume the same secret material.
 
 Long-lived runtime state is now sourced from age-encrypted files:
 
+- `secrets/forgejo/admin-password.age`
+- `secrets/forgejo/agent-ssh-key.age`
 - `secrets/forgejo/nsc-token.age`
 - `secrets/forgejo/nsc-dispatcher-config.age`
 - `secrets/forgejo/nsc-autoscaler-config.age`
 
 After refreshing the intake files, re-encrypt them into `secrets/forgejo/*.age`
 and deploy the forge host so `config.age.secrets.*` updates the live paths for
+`services.burrow.forge`, `services.burrow.forgeRunner`, and
 `services.burrow.forgejoNsc`.
 
 Run it next to the dispatcher:

services/forgejo-nsc/cmd/forgejo-nsc-dispatcher/main.go

@@ -43,19 +43,23 @@ func main() {
 	}
 
 	dispatcher, err := nsc.NewDispatcher(nsc.Options{
-		BinaryPath:       cfg.Namespace.NSCBinary,
-		ComputeBaseURL:   cfg.Namespace.ComputeBaseURL,
-		DefaultImage:     cfg.Namespace.Image,
-		DefaultMachine:   cfg.Namespace.MachineType,
-		MacosBaseImageID: cfg.Namespace.MacosBaseImageID,
-		MacosMachineArch: cfg.Namespace.MacosMachineArch,
-		DefaultDuration:  cfg.Namespace.Duration.Duration,
-		WorkDir:          cfg.Namespace.WorkDir,
-		MaxParallel:      cfg.Namespace.MaxParallel,
-		RunnerNamePrefix: cfg.Runner.NamePrefix,
-		Executor:         cfg.Runner.Executor,
-		Network:          cfg.Namespace.Network,
-		Logger:           logger,
+		BinaryPath:        cfg.Namespace.NSCBinary,
+		ComputeBaseURL:    cfg.Namespace.ComputeBaseURL,
+		DefaultImage:      cfg.Namespace.Image,
+		DefaultMachine:    cfg.Namespace.MachineType,
+		MacosBaseImageID:  cfg.Namespace.MacosBaseImageID,
+		MacosMachineArch:  cfg.Namespace.MacosMachineArch,
+		DefaultDuration:   cfg.Namespace.Duration.Duration,
+		WorkDir:           cfg.Namespace.WorkDir,
+		MaxParallel:       cfg.Namespace.MaxParallel,
+		LinuxCachePath:    cfg.Namespace.LinuxCachePath,
+		LinuxCacheVolumes: toNSCCacheVolumes(cfg.Namespace.LinuxCacheVolumes),
+		MacosCachePath:    cfg.Namespace.MacosCachePath,
+		MacosCacheVolumes: toNSCCacheVolumes(cfg.Namespace.MacosCacheVolumes),
+		RunnerNamePrefix:  cfg.Runner.NamePrefix,
+		Executor:          cfg.Runner.Executor,
+		Network:           cfg.Namespace.Network,
+		Logger:            logger,
 	})
 	if err != nil {
 		logger.Error("failed to create dispatcher", "error", err)
@@ -88,3 +92,15 @@ func main() {
 	defer cancel()
 	_ = srv.Shutdown(ctx)
 }
+
+func toNSCCacheVolumes(volumes []config.CacheVolumeConfig) []nsc.CacheVolume {
+	out := make([]nsc.CacheVolume, 0, len(volumes))
+	for _, volume := range volumes {
+		out = append(out, nsc.CacheVolume{
+			Tag:        volume.Tag,
+			MountPoint: volume.MountPoint,
+			SizeGb:     volume.SizeGb,
+		})
+	}
+	return out
+}

services/forgejo-nsc/config.example.yaml

@@ -21,6 +21,19 @@ namespace:
   workdir: "/var/lib/forgejo-runner"
   max_parallel: 4
   network: ""
+  linux_cache_path: "/var/cache/burrow"
+  linux_cache_volumes:
+    - tag: "burrow-forgejo-linux-nix"
+      mount_point: "/nix"
+      size_gb: 60
+    - tag: "burrow-forgejo-linux-cache"
+      mount_point: "/var/cache/burrow"
+      size_gb: 40
+  macos_cache_path: "/Users/runner/.cache/burrow"
+  macos_cache_volumes:
+    - tag: "burrow-forgejo-macos-cache"
+      mount_point: "/Users/runner/.cache/burrow"
+      size_gb: 60
 
 runner:
   name_prefix: "nscloud-"

services/forgejo-nsc/deploy/dispatcher.yaml

@@ -31,6 +31,19 @@ namespace:
   instance_tags:
     - "burrow"
   network: ""
+  linux_cache_path: "/var/cache/burrow"
+  linux_cache_volumes:
+    - tag: "burrow-forgejo-linux-nix"
+      mount_point: "/nix"
+      size_gb: 60
+    - tag: "burrow-forgejo-linux-cache"
+      mount_point: "/var/cache/burrow"
+      size_gb: 40
+  macos_cache_path: "/Users/runner/.cache/burrow"
+  macos_cache_volumes:
+    - tag: "burrow-forgejo-macos-cache"
+      mount_point: "/Users/runner/.cache/burrow"
+      size_gb: 60
 
 runner:
   name_prefix: "nscloud-"

services/forgejo-nsc/internal/config/config.go

@@ -49,8 +49,14 @@ type Config struct {
 	Runner    RunnerConfig    `yaml:"runner"`
 }
 
+type CacheVolumeConfig struct {
+	Tag        string `yaml:"tag"`
+	MountPoint string `yaml:"mount_point"`
+	SizeGb     int64  `yaml:"size_gb"`
+}
+
 type ForgejoConfig struct {
-	BaseURL       string      `yaml:"base_url"`
+	BaseURL string `yaml:"base_url"`
 	// InstanceURL is the URL runners should use when registering with Forgejo.
 	// This must be reachable from the spawned runner (e.g. the public URL like
 	// https://git.burrow.net), and may differ from BaseURL (which can be a local
@@ -80,15 +86,19 @@ type NamespaceConfig struct {
 	// MacosBaseImageID selects which macOS base image to use (e.g. "tahoe").
 	MacosBaseImageID string `yaml:"macos_base_image_id"`
 	// MacosMachineArch is the architecture used for macOS instances (typically "arm64").
-	MacosMachineArch string   `yaml:"macos_machine_arch"`
-	Duration         Duration `yaml:"duration"`
-	WorkDir          string   `yaml:"workdir"`
-	MaxParallel      int64    `yaml:"max_parallel"`
-	Environment      []string `yaml:"environment"`
-	AllowLabels      []string `yaml:"allow_labels"`
-	AllowScopes      []string `yaml:"allow_scopes"`
-	Network          string   `yaml:"network"`
-	InstanceTags     []string `yaml:"instance_tags"`
+	MacosMachineArch  string              `yaml:"macos_machine_arch"`
+	Duration          Duration            `yaml:"duration"`
+	WorkDir           string              `yaml:"workdir"`
+	MaxParallel       int64               `yaml:"max_parallel"`
+	Environment       []string            `yaml:"environment"`
+	AllowLabels       []string            `yaml:"allow_labels"`
+	AllowScopes       []string            `yaml:"allow_scopes"`
+	Network           string              `yaml:"network"`
+	InstanceTags      []string            `yaml:"instance_tags"`
+	LinuxCachePath    string              `yaml:"linux_cache_path"`
+	LinuxCacheVolumes []CacheVolumeConfig `yaml:"linux_cache_volumes"`
+	MacosCachePath    string              `yaml:"macos_cache_path"`
+	MacosCacheVolumes []CacheVolumeConfig `yaml:"macos_cache_volumes"`
 }
 
 type RunnerConfig struct {
@@ -160,6 +170,46 @@ func (c *Config) Validate() error {
 	if c.Namespace.MaxParallel <= 0 {
 		c.Namespace.MaxParallel = 4
 	}
+	if c.Namespace.LinuxCachePath == "" {
+		c.Namespace.LinuxCachePath = "/var/cache/burrow"
+	}
+	if len(c.Namespace.LinuxCacheVolumes) == 0 {
+		c.Namespace.LinuxCacheVolumes = []CacheVolumeConfig{
+			{
+				Tag:        "burrow-forgejo-linux-nix",
+				MountPoint: "/nix",
+				SizeGb:     60,
+			},
+			{
+				Tag:        "burrow-forgejo-linux-cache",
+				MountPoint: c.Namespace.LinuxCachePath,
+				SizeGb:     40,
+			},
+		}
+	}
+	if c.Namespace.MacosCachePath == "" {
+		c.Namespace.MacosCachePath = "/Users/runner/.cache/burrow"
+	}
+	if len(c.Namespace.MacosCacheVolumes) == 0 {
+		c.Namespace.MacosCacheVolumes = []CacheVolumeConfig{
+			{
+				Tag:        "burrow-forgejo-macos-cache",
+				MountPoint: c.Namespace.MacosCachePath,
+				SizeGb:     60,
+			},
+		}
+	}
+	for _, volume := range append(append([]CacheVolumeConfig{}, c.Namespace.LinuxCacheVolumes...), c.Namespace.MacosCacheVolumes...) {
+		if strings.TrimSpace(volume.Tag) == "" {
+			return errors.New("namespace cache volume tag is required")
+		}
+		if strings.TrimSpace(volume.MountPoint) == "" {
+			return fmt.Errorf("namespace cache volume %q mount_point is required", volume.Tag)
+		}
+		if volume.SizeGb <= 0 {
+			return fmt.Errorf("namespace cache volume %q size_gb must be positive", volume.Tag)
+		}
+	}
 
 	return nil
 }

services/forgejo-nsc/internal/nsc/dispatcher.go

@@ -17,19 +17,29 @@ import (
 )
 
 type Options struct {
-	BinaryPath       string
-	DefaultImage     string
-	DefaultMachine   string
-	DefaultDuration  time.Duration
-	WorkDir          string
-	MaxParallel      int64
-	RunnerNamePrefix string
-	Executor         string
-	Network          string
-	ComputeBaseURL   string
-	MacosBaseImageID string
-	MacosMachineArch string
-	Logger           *slog.Logger
+	BinaryPath        string
+	DefaultImage      string
+	DefaultMachine    string
+	DefaultDuration   time.Duration
+	WorkDir           string
+	MaxParallel       int64
+	RunnerNamePrefix  string
+	Executor          string
+	Network           string
+	ComputeBaseURL    string
+	MacosBaseImageID  string
+	MacosMachineArch  string
+	LinuxCachePath    string
+	LinuxCacheVolumes []CacheVolume
+	MacosCachePath    string
+	MacosCacheVolumes []CacheVolume
+	Logger            *slog.Logger
+}
+
+type CacheVolume struct {
+	Tag        string
+	MountPoint string
+	SizeGb     int64
 }
 
 type LaunchRequest struct {
@@ -73,6 +83,12 @@ func NewDispatcher(opts Options) (*Dispatcher, error) {
 	if opts.DefaultDuration == 0 {
 		opts.DefaultDuration = 30 * time.Minute
 	}
+	if opts.LinuxCachePath == "" {
+		opts.LinuxCachePath = "/var/cache/burrow"
+	}
+	if opts.MacosCachePath == "" {
+		opts.MacosCachePath = "/Users/runner/.cache/burrow"
+	}
 	logger := opts.Logger
 	if logger == nil {
 		logger = slog.New(slog.NewTextHandler(io.Discard, nil))
@@ -104,6 +120,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
 	}
 	machineType := choose(req.MachineType, d.opts.DefaultMachine)
 	image := choose(req.Image, d.opts.DefaultImage)
+	if req.ExtraEnv == nil {
+		req.ExtraEnv = make(map[string]string)
+	}
 
 	if hasWindowsLabel(req.Labels) {
 		if err := d.launchWindowsRunnerViaWinRM(ctx, runnerName, req, duration, machineType); err != nil {
@@ -113,6 +132,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
 	}
 
 	if hasMacOSLabel(req.Labels) {
+		if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok {
+			req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.MacosCachePath
+		}
 		// Compute macOS shapes differ from the Linux "run" defaults. If the request
 		// didn't specify a machine type, ensure we pick a macOS-valid default.
 		if machineType == "" || machineType == d.opts.DefaultMachine {
@@ -129,6 +151,9 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
 		}
 		return runnerName, nil
 	}
+	if _, ok := req.ExtraEnv["NSC_CACHE_PATH"]; !ok {
+		req.ExtraEnv["NSC_CACHE_PATH"] = d.opts.LinuxCachePath
+	}
 
 	env := map[string]string{
 		"FORGEJO_INSTANCE_URL":  req.InstanceURL,
@@ -140,9 +165,6 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
 	for k, v := range req.ExtraEnv {
 		env[k] = v
 	}
-	if _, ok := env["NSC_CACHE_PATH"]; !ok {
-		env["NSC_CACHE_PATH"] = "/nix/store"
-	}
 
 	script := d.bootstrapScript()
 	args := []string{
@@ -161,6 +183,7 @@ func (d *Dispatcher) LaunchRunner(ctx context.Context, req LaunchRequest) (strin
 	if d.opts.Network != "" {
 		args = append(args, "--network", d.opts.Network)
 	}
+	args = appendVolumeArgs(args, d.opts.LinuxCacheVolumes)
 	for key, value := range env {
 		if value == "" {
 			continue
@@ -370,6 +393,16 @@ func choose(values ...string) string {
 	return ""
 }
 
+func appendVolumeArgs(args []string, volumes []CacheVolume) []string {
+	for _, volume := range volumes {
+		if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 {
+			continue
+		}
+		args = append(args, "--volume", fmt.Sprintf("cache:%s:%s:%d", volume.Tag, volume.MountPoint, volume.SizeGb))
+	}
+	return args
+}
+
 func (d *Dispatcher) bootstrapScript() string {
 	var builder strings.Builder
 	builder.WriteString(`set -euo pipefail

services/forgejo-nsc/internal/nsc/macos.go

@@ -206,12 +206,8 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r
 	for k, v := range req.ExtraEnv {
 		env[k] = v
 	}
-	// Best-effort caching: workflows call Scripts/nscloud-cache.sh, which is a
-	// no-op unless NSC_CACHE_PATH is set. This may still be skipped if spacectl
-	// lacks credentials, but setting the path is harmless and keeps behavior
-	// consistent across macOS / Linux runners.
 	if _, ok := env["NSC_CACHE_PATH"]; !ok {
-		env["NSC_CACHE_PATH"] = "/Users/runner/.cache/nscloud"
+		env["NSC_CACHE_PATH"] = d.opts.MacosCachePath
 	}
 
 	deadline := timestamppb.New(time.Now().Add(ttl))
@@ -243,10 +239,15 @@ func (d *Dispatcher) launchMacOSRunner(ctx context.Context, runnerName string, r
 			},
 		},
 	}
+	experimental := &computev1beta.CreateInstanceRequest_ExperimentalFeatures{}
 	if imageID := macosComputeBaseImageID(d.opts.MacosBaseImageID); imageID != "" {
-		createReq.Experimental = &computev1beta.CreateInstanceRequest_ExperimentalFeatures{
-			MacosBaseImageId: imageID,
-		}
+		experimental.MacosBaseImageId = imageID
+	}
+	if volumes := computeCacheVolumeRequests(d.opts.MacosCacheVolumes); len(volumes) > 0 {
+		experimental.Volumes = volumes
+	}
+	if experimental.MacosBaseImageId != "" || len(experimental.Volumes) > 0 {
+		createReq.Experimental = experimental
 	}
 
 	d.log.Info("launching Namespace macos runner",
@@ -572,6 +573,22 @@ func (d *Dispatcher) destroyComputeInstance(ctx context.Context, client computev
 	d.log.Info("macos runner destroyed", "runner", runnerName, "instance", instanceID)
 }
 
+func computeCacheVolumeRequests(volumes []CacheVolume) []*computev1beta.VolumeRequest {
+	var out []*computev1beta.VolumeRequest
+	for _, volume := range volumes {
+		if strings.TrimSpace(volume.Tag) == "" || strings.TrimSpace(volume.MountPoint) == "" || volume.SizeGb <= 0 {
+			continue
+		}
+		out = append(out, &computev1beta.VolumeRequest{
+			MountPoint:      volume.MountPoint,
+			Tag:             volume.Tag,
+			SizeMb:          volume.SizeGb * 1024,
+			PersistencyKind: computev1beta.VolumeRequest_CACHE,
+		})
+	}
+	return out
+}
+
 func macosBootstrapScript() string {
 	// Keep this script self-contained: it runs on a fresh macOS VM base image.
 	var b strings.Builder

services/forgejo-nsc/internal/nsc/macos_nsc.go

@@ -144,6 +144,7 @@ func (d *Dispatcher) launchMacOSRunnerViaNSC(ctx context.Context, runnerName str
 			"--wait_timeout", a.waitTimeout.String(),
 		}
 		args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
+		args = appendVolumeArgs(args, d.opts.MacosCacheVolumes)
 
 		createCtx, cancel := context.WithTimeout(ctx, a.createTimeout)
 		defer cancel()