Use upstream nsc-autoscaler on burrow forge
This commit is contained in:
Conrad Kramer
parent
3d80e772c8
commit
9e3e8fa783
6 changed files with 36 additions and 241 deletions
26
flake.lock
generated
26
flake.lock
generated
|
|
@ -123,13 +123,37 @@
|
|||
"url": "https://codeload.github.com/NixOS/nixpkgs/tar.gz/nixos-unstable"
|
||||
}
|
||||
},
|
||||
"nsc-autoscaler": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1775221037,
|
||||
"narHash": "sha256-tv6Y3cqn76PEyZpSMMItVW96KKIboovBWTOv5Lt7PXg=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "2c485752fde28ec3be2f228b571d1906f4bcf917",
|
||||
"revCount": 10,
|
||||
"type": "git",
|
||||
"url": "https://compatible.systems/conrad/nsc-autoscaler.git"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://compatible.systems/conrad/nsc-autoscaler.git"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"agenix": "agenix",
|
||||
"disko": "disko",
|
||||
"flake-utils": "flake-utils",
|
||||
"hcloud-upload-image-src": "hcloud-upload-image-src",
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nsc-autoscaler": "nsc-autoscaler"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
|
|
|
|||
|
|
@ -12,13 +12,18 @@
|
|||
url = "tarball+https://codeload.github.com/nix-community/disko/tar.gz/master";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
nsc-autoscaler = {
|
||||
url = "git+https://compatible.systems/conrad/nsc-autoscaler.git";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
};
|
||||
hcloud-upload-image-src = {
|
||||
url = "tarball+https://codeload.github.com/apricote/hcloud-upload-image/tar.gz/v1.3.0";
|
||||
flake = false;
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils, agenix, disko, hcloud-upload-image-src }:
|
||||
outputs = { self, nixpkgs, flake-utils, agenix, disko, nsc-autoscaler, hcloud-upload-image-src }:
|
||||
let
|
||||
supportedSystems = [
|
||||
"x86_64-linux"
|
||||
|
|
@ -175,7 +180,7 @@
|
|||
// {
|
||||
nixosModules.burrow-forge = import ./nixos/modules/burrow-forge.nix;
|
||||
nixosModules.burrow-forge-runner = import ./nixos/modules/burrow-forge-runner.nix;
|
||||
nixosModules.burrow-forgejo-nsc = import ./nixos/modules/burrow-forgejo-nsc.nix;
|
||||
nixosModules.burrow-forgejo-nsc = nsc-autoscaler.nixosModules.default;
|
||||
nixosModules.burrow-authentik = import ./nixos/modules/burrow-authentik.nix;
|
||||
nixosModules.burrow-headscale = import ./nixos/modules/burrow-headscale.nix;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Mail hosting is intentionally not part of this NixOS host in the current plan. B
|
|||
- `hosts/burrow-forge/default.nix`: host entrypoint
|
||||
- `modules/burrow-forge.nix`: Forgejo, Caddy, PostgreSQL, and admin bootstrap module
|
||||
- `modules/burrow-forge-runner.nix`: Forgejo Actions runner and agent identity bootstrap
|
||||
- `modules/burrow-forgejo-nsc.nix`: Namespace-backed ephemeral Forgejo runner services
|
||||
- upstream `compatible.systems/conrad/nsc-autoscaler`: Namespace-backed ephemeral Forgejo runner module consumed via the Burrow flake input
|
||||
- `modules/burrow-authentik.nix`: minimal Authentik IdP for Burrow control planes
|
||||
- `modules/burrow-headscale.nix`: Headscale control plane rooted in Authentik OIDC
|
||||
- `../secrets.nix`: agenix recipient map for tracked Burrow forge secrets
|
||||
|
|
@ -32,7 +32,7 @@ Mail hosting is intentionally not part of this NixOS host in the current plan. B
|
|||
3. Run `Scripts/bootstrap-forge-intake.sh` to place the Forgejo bootstrap password file and automation SSH key under `/var/lib/burrow/intake/`.
|
||||
4. Let `burrow-forgejo-bootstrap.service` create or rotate the initial Forgejo admin account.
|
||||
5. Let `burrow-forgejo-runner-bootstrap.service` register the self-hosted Forgejo runner and seed Git identity as `agent <agent@burrow.net>`.
|
||||
6. Run `Scripts/provision-forgejo-nsc.sh` locally, then `Scripts/sync-forgejo-nsc-config.sh` to place the Namespace dispatcher/autoscaler runtime inputs under `/var/lib/burrow/intake/`.
|
||||
6. Run `Scripts/provision-forgejo-nsc.sh` locally, then `Scripts/sync-forgejo-nsc-config.sh` to place the raw Namespace dispatcher/autoscaler runtime inputs under `/var/lib/burrow/intake/` for the upstream `services.forgejo-nsc` module.
|
||||
7. Ensure `/var/lib/agenix/agenix.key` exists on the host, encrypt `secrets/infra/authentik.env.age`, `secrets/infra/authentik-google-client-id.age`, `secrets/infra/authentik-google-client-secret.age`, `secrets/infra/forgejo-oidc-client-secret.age`, and `secrets/infra/headscale-oidc-client-secret.age`, and let agenix materialize them under `/run/agenix/`.
|
||||
8. Use `Scripts/cloudflare-upsert-a-record.sh` to point `git.burrow.net`, `burrow.net`, `auth.burrow.net`, `ts.burrow.net`, and `nsc-autoscaler.burrow.net` at the host with Cloudflare proxying disabled for ACME.
|
||||
9. Use `Scripts/forge-deploy.sh --allow-dirty` for subsequent remote `nixos-rebuild` runs from the live workspace.
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ in
|
|||
sshPrivateKeyFile = "/var/lib/burrow/intake/agent_at_burrow_net_ed25519";
|
||||
};
|
||||
|
||||
services.burrow.forgejoNsc = {
|
||||
services.forgejo-nsc = {
|
||||
enable = true;
|
||||
nscTokenFile = "/var/lib/burrow/intake/forgejo_nsc_token.txt";
|
||||
dispatcher = {
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@ in
|
|||
'';
|
||||
}
|
||||
// lib.optionalAttrs (
|
||||
config.services.burrow.forgejoNsc.enable && config.services.burrow.forgejoNsc.autoscaler.enable
|
||||
config.services.forgejo-nsc.enable && config.services.forgejo-nsc.autoscaler.enable
|
||||
) {
|
||||
"${cfg.nscAutoscalerDomain}".extraConfig = ''
|
||||
encode gzip zstd
|
||||
|
|
|
|||
|
|
@ -1,234 +0,0 @@
|
|||
{ config, lib, pkgs, self, ... }:
|
||||
|
||||
let
|
||||
inherit (lib)
|
||||
mkEnableOption
|
||||
mkIf
|
||||
mkOption
|
||||
types
|
||||
mkAfter
|
||||
mkDefault
|
||||
optional
|
||||
optionalAttrs
|
||||
optionalString
|
||||
;
|
||||
|
||||
cfg = config.services.burrow.forgejoNsc;
|
||||
dispatcherRuntimeConfig = "${cfg.stateDir}/dispatcher.yaml";
|
||||
autoscalerRuntimeConfig = "${cfg.stateDir}/autoscaler.yaml";
|
||||
|
||||
pendingCheck = configPath: pkgs.writeShellScript "forgejo-nsc-check-pending" ''
|
||||
set -euo pipefail
|
||||
if ${pkgs.gnugrep}/bin/grep -q 'PENDING-' '${configPath}'; then
|
||||
echo "forgejo-nsc config still contains placeholder values (PENDING-); update ${configPath} before starting." >&2
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
|
||||
nscTokenPath = "${cfg.stateDir}/nsc.token";
|
||||
tokenSync = optionalString (cfg.nscTokenFile != null) ''
|
||||
install -m 600 ${lib.escapeShellArg cfg.nscTokenFile} ${lib.escapeShellArg nscTokenPath}
|
||||
chown ${cfg.user}:${cfg.group} ${nscTokenPath}
|
||||
chmod 600 ${nscTokenPath}
|
||||
'';
|
||||
dispatcherConfigSync = optionalString (cfg.dispatcher.configFile != null) ''
|
||||
install -m 400 ${lib.escapeShellArg cfg.dispatcher.configFile} ${lib.escapeShellArg dispatcherRuntimeConfig}
|
||||
chown ${cfg.user}:${cfg.group} ${lib.escapeShellArg dispatcherRuntimeConfig}
|
||||
chmod 400 ${lib.escapeShellArg dispatcherRuntimeConfig}
|
||||
'';
|
||||
autoscalerConfigSync = optionalString (cfg.autoscaler.configFile != null) ''
|
||||
install -m 400 ${lib.escapeShellArg cfg.autoscaler.configFile} ${lib.escapeShellArg autoscalerRuntimeConfig}
|
||||
chown ${cfg.user}:${cfg.group} ${lib.escapeShellArg autoscalerRuntimeConfig}
|
||||
chmod 400 ${lib.escapeShellArg autoscalerRuntimeConfig}
|
||||
'';
|
||||
|
||||
dispatcherEnv =
|
||||
cfg.extraEnv
|
||||
// optionalAttrs (cfg.nscTokenFile != null) { NSC_TOKEN_FILE = nscTokenPath; }
|
||||
// optionalAttrs (cfg.nscTokenSpecFile != null) { NSC_TOKEN_SPEC_FILE = cfg.nscTokenSpecFile; }
|
||||
// optionalAttrs (cfg.nscEndpoint != null) { NSC_ENDPOINT = cfg.nscEndpoint; };
|
||||
in {
|
||||
options.services.burrow.forgejoNsc = {
|
||||
enable = mkEnableOption "Forgejo Namespace Cloud runner dispatcher";
|
||||
|
||||
user = mkOption {
|
||||
type = types.str;
|
||||
default = "forgejo-nsc";
|
||||
description = "System user that runs the forgejo-nsc services.";
|
||||
};
|
||||
|
||||
group = mkOption {
|
||||
type = types.str;
|
||||
default = "forgejo-nsc";
|
||||
description = "System group for the forgejo-nsc services.";
|
||||
};
|
||||
|
||||
stateDir = mkOption {
|
||||
type = types.str;
|
||||
default = "/var/lib/forgejo-nsc";
|
||||
description = "State directory for the dispatcher/autoscaler.";
|
||||
};
|
||||
|
||||
nscTokenFile = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Optional NSC token file (exported as NSC_TOKEN_FILE).";
|
||||
};
|
||||
|
||||
nscTokenSpecFile = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Optional NSC token spec file (exported as NSC_TOKEN_SPEC_FILE).";
|
||||
};
|
||||
|
||||
nscEndpoint = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Optional NSC endpoint override (exported as NSC_ENDPOINT).";
|
||||
};
|
||||
|
||||
extraEnv = mkOption {
|
||||
type = types.attrsOf types.str;
|
||||
default = { };
|
||||
description = "Extra environment variables injected into the services.";
|
||||
};
|
||||
|
||||
nscPackage = mkOption {
|
||||
type = types.nullOr types.package;
|
||||
default = self.packages.${pkgs.stdenv.hostPlatform.system}.nsc or null;
|
||||
description = "Optional nsc CLI package added to the service PATH.";
|
||||
};
|
||||
|
||||
dispatcher = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Enable the forgejo-nsc dispatcher service.";
|
||||
};
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = self.packages.${pkgs.stdenv.hostPlatform.system}.forgejo-nsc-dispatcher;
|
||||
description = "Package providing the forgejo-nsc dispatcher binary.";
|
||||
};
|
||||
|
||||
configFile = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Host-local YAML config file for the dispatcher.";
|
||||
};
|
||||
|
||||
allowPending = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Allow placeholder values (PENDING-) in the dispatcher config.";
|
||||
};
|
||||
};
|
||||
|
||||
autoscaler = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Enable the forgejo-nsc autoscaler service.";
|
||||
};
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = self.packages.${pkgs.stdenv.hostPlatform.system}.forgejo-nsc-autoscaler;
|
||||
description = "Package providing the forgejo-nsc autoscaler binary.";
|
||||
};
|
||||
|
||||
configFile = mkOption {
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
description = "Host-local YAML config file for the autoscaler.";
|
||||
};
|
||||
|
||||
allowPending = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Allow placeholder values (PENDING-) in the autoscaler config.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = (!cfg.dispatcher.enable) || cfg.dispatcher.configFile != null;
|
||||
message = "services.burrow.forgejoNsc.dispatcher.configFile must be set when the dispatcher is enabled.";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.autoscaler.enable) || cfg.autoscaler.configFile != null;
|
||||
message = "services.burrow.forgejoNsc.autoscaler.configFile must be set when the autoscaler is enabled.";
|
||||
}
|
||||
];
|
||||
|
||||
users.groups.${cfg.group} = { };
|
||||
users.users.${cfg.user} = {
|
||||
uid = mkDefault 2011;
|
||||
isSystemUser = true;
|
||||
group = cfg.group;
|
||||
description = "Forgejo Namespace Cloud runner services";
|
||||
home = cfg.stateDir;
|
||||
createHome = true;
|
||||
shell = pkgs.bashInteractive;
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = mkAfter [
|
||||
"d ${cfg.stateDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
];
|
||||
|
||||
systemd.services.forgejo-nsc-dispatcher = mkIf cfg.dispatcher.enable {
|
||||
description = "Forgejo Namespace Cloud dispatcher";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
unitConfig.ConditionPathExists =
|
||||
optional (cfg.dispatcher.configFile != null) cfg.dispatcher.configFile
|
||||
++ optional (cfg.nscTokenFile != null) cfg.nscTokenFile;
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
WorkingDirectory = cfg.stateDir;
|
||||
ExecStart = "${cfg.dispatcher.package}/bin/forgejo-nsc-dispatcher --config ${dispatcherRuntimeConfig}";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
};
|
||||
path = lib.optional (cfg.nscPackage != null) cfg.nscPackage;
|
||||
environment = dispatcherEnv;
|
||||
preStart = lib.concatStringsSep "\n" (lib.filter (s: s != "") [
|
||||
(optionalString (!cfg.dispatcher.allowPending) (pendingCheck cfg.dispatcher.configFile))
|
||||
dispatcherConfigSync
|
||||
tokenSync
|
||||
]);
|
||||
};
|
||||
|
||||
systemd.services.forgejo-nsc-autoscaler = mkIf cfg.autoscaler.enable {
|
||||
description = "Forgejo Namespace Cloud autoscaler";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "forgejo-nsc-dispatcher.service" ];
|
||||
wants = [ "network-online.target" ];
|
||||
unitConfig.ConditionPathExists =
|
||||
optional (cfg.autoscaler.configFile != null) cfg.autoscaler.configFile
|
||||
++ optional (cfg.nscTokenFile != null) cfg.nscTokenFile;
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
WorkingDirectory = cfg.stateDir;
|
||||
ExecStart = "${cfg.autoscaler.package}/bin/forgejo-nsc-autoscaler --config ${autoscalerRuntimeConfig}";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
};
|
||||
path = lib.optional (cfg.nscPackage != null) cfg.nscPackage;
|
||||
environment = dispatcherEnv;
|
||||
preStart = lib.concatStringsSep "\n" (lib.filter (s: s != "") [
|
||||
(optionalString (!cfg.autoscaler.allowPending) (pendingCheck cfg.autoscaler.configFile))
|
||||
autoscalerConfigSync
|
||||
tokenSync
|
||||
]);
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue