Add Burrow forge infrastructure and tailnet control plane
This commit is contained in:
Conrad Kramer
parent
d1ed826389
commit
de25f240d5
51 changed files with 9058 additions and 0 deletions
499
services/forgejo-nsc/internal/nsc/windows_winrm.go
Normal file
499
services/forgejo-nsc/internal/nsc/windows_winrm.go
Normal file
|
|
@ -0,0 +1,499 @@
|
|||
package nsc
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type windowsProxyOutput struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
RDP struct {
|
||||
Credentials struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
} `json:"credentials"`
|
||||
} `json:"rdp"`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) launchWindowsRunnerViaWinRM(ctx context.Context, runnerName string, req LaunchRequest, ttl time.Duration, machineType string) error {
|
||||
script := windowsBootstrapScript(runnerName, req, d.opts.Executor, d.opts.WorkDir)
|
||||
return d.launchWindowsScriptViaWinRM(ctx, runnerName, ttl, machineType, req.Labels, script)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) launchWindowsScriptViaWinRM(ctx context.Context, runnerName string, ttl time.Duration, machineType string, labels []string, script string) error {
|
||||
if ttl <= 0 {
|
||||
ttl = d.opts.DefaultDuration
|
||||
}
|
||||
|
||||
mt := normalizeWindowsMachineType(machineType, labels)
|
||||
instanceID, createOutput, err := d.createWindowsInstance(ctx, runnerName, ttl, mt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("windows create failed: %w\n%s", err, createOutput)
|
||||
}
|
||||
defer d.destroyNSCInstance(context.Background(), runnerName, instanceID)
|
||||
|
||||
username, password, err := d.resolveWindowsCredentials(ctx, instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := d.probeWindowsWinRMService(ctx, instanceID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
endpoint, stopForward, err := d.startWindowsWinRMPortForward(ctx, instanceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopForward()
|
||||
|
||||
if err := d.runWindowsWinRMPowerShell(ctx, endpoint, username, password, script); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) createWindowsInstance(ctx context.Context, runnerName string, ttl time.Duration, machineType string) (instanceID string, output string, err error) {
|
||||
tmpDir, err := os.MkdirTemp("", "forgejo-nsc-windows-*")
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("mktemp: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
metaPath := filepath.Join(tmpDir, "create.json")
|
||||
cidPath := filepath.Join(tmpDir, "create.cid")
|
||||
|
||||
args := []string{
|
||||
"create",
|
||||
"--duration", ttl.String(),
|
||||
"--machine_type", machineType,
|
||||
"--cidfile", cidPath,
|
||||
"--purpose", fmt.Sprintf("burrow forgejo runner %s", runnerName),
|
||||
"--output", "plain",
|
||||
"--output_json_to", metaPath,
|
||||
"--wait_timeout", "6m",
|
||||
}
|
||||
args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
|
||||
|
||||
createCtx, cancel := context.WithTimeout(ctx, 8*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(createCtx, d.opts.BinaryPath, args...)
|
||||
var buf bytes.Buffer
|
||||
cmd.Stdout = &buf
|
||||
cmd.Stderr = &buf
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
if created := strings.TrimSpace(mustReadFile(cidPath)); created != "" {
|
||||
d.destroyNSCInstance(context.Background(), runnerName, created)
|
||||
}
|
||||
if errors.Is(createCtx.Err(), context.DeadlineExceeded) {
|
||||
return "", buf.String(), fmt.Errorf("nsc create timed out after %s", 8*time.Minute)
|
||||
}
|
||||
return "", buf.String(), fmt.Errorf("nsc create failed: %w", err)
|
||||
}
|
||||
|
||||
instanceID, err = readNSCCreateInstanceID(metaPath)
|
||||
if err != nil {
|
||||
return "", buf.String(), fmt.Errorf("nsc create output parse failed: %w", err)
|
||||
}
|
||||
if instanceID == "" {
|
||||
return "", buf.String(), errors.New("nsc create returned empty instance id")
|
||||
}
|
||||
return instanceID, buf.String(), nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) resolveWindowsCredentials(ctx context.Context, instanceID string) (username string, password string, err error) {
|
||||
tmpDir, err := os.MkdirTemp("", "forgejo-nsc-winproxy-*")
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("mktemp: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
outPath := filepath.Join(tmpDir, "proxy.json")
|
||||
outFile, err := os.Create(outPath)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("create proxy output file: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
var stderr bytes.Buffer
|
||||
args := []string{"instance", "proxy", instanceID, "-s", "rdp", "-o", "json"}
|
||||
args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
|
||||
|
||||
proxyCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(proxyCtx, d.opts.BinaryPath, args...)
|
||||
cmd.Stdout = outFile
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", "", fmt.Errorf("start nsc instance proxy: %w", err)
|
||||
}
|
||||
|
||||
waitDone := make(chan struct{})
|
||||
var waitErr error
|
||||
go func() {
|
||||
waitErr = cmd.Wait()
|
||||
close(waitDone)
|
||||
}()
|
||||
|
||||
var payload windowsProxyOutput
|
||||
deadline := time.Now().Add(45 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
raw, _ := os.ReadFile(outPath)
|
||||
jsonBlob := extractJSON(string(raw))
|
||||
if jsonBlob != "" {
|
||||
if err := json.Unmarshal([]byte(jsonBlob), &payload); err == nil {
|
||||
username = strings.TrimSpace(payload.RDP.Credentials.Username)
|
||||
password = strings.TrimSpace(payload.RDP.Credentials.Password)
|
||||
if username != "" && password != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-waitDone:
|
||||
if waitErr != nil {
|
||||
return "", "", fmt.Errorf("nsc instance proxy exited before credentials were available: %w\n%s", waitErr, stderr.String())
|
||||
}
|
||||
default:
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
if cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
<-waitDone
|
||||
|
||||
if username == "" || password == "" {
|
||||
raw, _ := os.ReadFile(outPath)
|
||||
return "", "", fmt.Errorf("failed to resolve windows credentials from nsc instance proxy output\nstdout=%s\nstderr=%s", strings.TrimSpace(string(raw)), strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return username, password, nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) probeWindowsWinRMService(ctx context.Context, instanceID string) error {
|
||||
args := []string{"instance", "proxy", instanceID, "-s", "winrm", "-o", "json", "--once"}
|
||||
args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(probeCtx, d.opts.BinaryPath, args...)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
|
||||
err := cmd.Run()
|
||||
raw := strings.TrimSpace(out.String())
|
||||
if endpoint, ok := parseProxyEndpoint(raw); ok && endpoint != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if indicatesMissingProxyService(raw, "winrm") {
|
||||
return fmt.Errorf("namespace windows non-interactive channel unavailable: instance does not expose winrm service (rdp-only)\n%s", raw)
|
||||
}
|
||||
|
||||
if errors.Is(probeCtx.Err(), context.DeadlineExceeded) {
|
||||
return fmt.Errorf("timed out probing Namespace winrm service before bootstrap\n%s", raw)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("nsc winrm service probe failed: %w\n%s", err, raw)
|
||||
}
|
||||
return fmt.Errorf("nsc winrm service probe did not yield endpoint output\n%s", raw)
|
||||
}
|
||||
|
||||
func parseProxyEndpoint(raw string) (string, bool) {
|
||||
jsonBlob := extractJSON(raw)
|
||||
if jsonBlob == "" {
|
||||
return "", false
|
||||
}
|
||||
var payload struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(jsonBlob), &payload); err != nil {
|
||||
return "", false
|
||||
}
|
||||
endpoint := strings.TrimSpace(payload.Endpoint)
|
||||
if endpoint == "" {
|
||||
return "", false
|
||||
}
|
||||
return endpoint, true
|
||||
}
|
||||
|
||||
func indicatesMissingProxyService(raw string, service string) bool {
|
||||
service = strings.TrimSpace(service)
|
||||
if service == "" {
|
||||
return false
|
||||
}
|
||||
token := fmt.Sprintf("does not have service %q", service)
|
||||
return strings.Contains(raw, token)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) startWindowsWinRMPortForward(ctx context.Context, instanceID string) (endpoint string, stop func(), err error) {
|
||||
args := []string{"instance", "port-forward", instanceID, "--target_port", "5985"}
|
||||
args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL)
|
||||
|
||||
forwardCtx, cancel := context.WithCancel(ctx)
|
||||
cmd := exec.CommandContext(forwardCtx, d.opts.BinaryPath, args...)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return "", nil, fmt.Errorf("port-forward stdout pipe: %w", err)
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
cancel()
|
||||
return "", nil, fmt.Errorf("start nsc port-forward: %w", err)
|
||||
}
|
||||
|
||||
waitDone := make(chan struct{})
|
||||
var waitErr error
|
||||
go func() {
|
||||
waitErr = cmd.Wait()
|
||||
close(waitDone)
|
||||
}()
|
||||
|
||||
endpointCh := make(chan string, 1)
|
||||
scanErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if strings.HasPrefix(line, "Listening on ") {
|
||||
endpointCh <- strings.TrimSpace(strings.TrimPrefix(line, "Listening on "))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
scanErrCh <- err
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case endpoint = <-endpointCh:
|
||||
stop = func() {
|
||||
cancel()
|
||||
if cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
<-waitDone
|
||||
}
|
||||
return endpoint, stop, nil
|
||||
case err := <-scanErrCh:
|
||||
cancel()
|
||||
if cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
<-waitDone
|
||||
return "", nil, fmt.Errorf("failed reading port-forward output: %w", err)
|
||||
case <-waitDone:
|
||||
cancel()
|
||||
if waitErr != nil {
|
||||
return "", nil, fmt.Errorf("nsc port-forward exited early: %w\n%s", waitErr, stderr.String())
|
||||
}
|
||||
return "", nil, fmt.Errorf("nsc port-forward exited without endpoint\n%s", stderr.String())
|
||||
case <-time.After(45 * time.Second):
|
||||
cancel()
|
||||
if cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
<-waitDone
|
||||
return "", nil, fmt.Errorf("timed out waiting for WinRM port-forward endpoint\n%s", stderr.String())
|
||||
case <-ctx.Done():
|
||||
cancel()
|
||||
if cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
<-waitDone
|
||||
return "", nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runWindowsWinRMPowerShell(ctx context.Context, endpoint, username, password, script string) error {
|
||||
pythonPath, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
return fmt.Errorf("python3 is required for windows WinRM bootstrap: %w", err)
|
||||
}
|
||||
|
||||
workdir := strings.TrimSpace(d.opts.WorkDir)
|
||||
if workdir == "" {
|
||||
workdir = "/tmp/forgejo-runner"
|
||||
}
|
||||
if err := os.MkdirAll(workdir, 0o755); err != nil {
|
||||
return fmt.Errorf("create workdir %s: %w", workdir, err)
|
||||
}
|
||||
|
||||
venvPath := filepath.Join(workdir, ".winrm-venv")
|
||||
venvPython := filepath.Join(venvPath, "bin", "python")
|
||||
if _, err := os.Stat(venvPython); err != nil {
|
||||
cmd := exec.CommandContext(ctx, pythonPath, "-m", "venv", venvPath)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("create python venv for winrm failed: %w\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
ensurePyWinRM := `
|
||||
import importlib.util, subprocess, sys
|
||||
if importlib.util.find_spec("winrm") is None:
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", "pywinrm"])
|
||||
`
|
||||
ensureCmd := exec.CommandContext(ctx, venvPython, "-c", ensurePyWinRM)
|
||||
var ensureOut bytes.Buffer
|
||||
ensureCmd.Stdout = &ensureOut
|
||||
ensureCmd.Stderr = &ensureOut
|
||||
if err := ensureCmd.Run(); err != nil {
|
||||
return fmt.Errorf("install pywinrm failed: %w\n%s", err, ensureOut.String())
|
||||
}
|
||||
|
||||
runScript := `
|
||||
import base64, os, sys, time, traceback, winrm
|
||||
|
||||
endpoint = os.environ["WINRM_ENDPOINT"]
|
||||
user = os.environ["WINRM_USER"]
|
||||
password = os.environ["WINRM_PASS"]
|
||||
script = base64.b64decode(os.environ["WINRM_SCRIPT_B64"]).decode("utf-8")
|
||||
|
||||
deadline = time.time() + 300.0
|
||||
last_err = None
|
||||
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
session = winrm.Session(f"http://{endpoint}/wsman", auth=(user, password), transport="ntlm")
|
||||
result = session.run_ps(script)
|
||||
sys.stdout.write(result.std_out.decode("utf-8", errors="replace"))
|
||||
sys.stderr.write(result.std_err.decode("utf-8", errors="replace"))
|
||||
print(f"winrm_exit={result.status_code}")
|
||||
sys.exit(result.status_code)
|
||||
except Exception as err:
|
||||
last_err = err
|
||||
time.sleep(5.0)
|
||||
|
||||
sys.stderr.write("timed out waiting for WinRM connectivity after 300s\\n")
|
||||
if last_err is not None:
|
||||
traceback.print_exception(last_err, file=sys.stderr)
|
||||
sys.exit(111)
|
||||
`
|
||||
runCmd := exec.CommandContext(ctx, venvPython, "-c", runScript)
|
||||
runCmd.Env = append(os.Environ(),
|
||||
"WINRM_ENDPOINT="+endpoint,
|
||||
"WINRM_USER="+username,
|
||||
"WINRM_PASS="+password,
|
||||
"WINRM_SCRIPT_B64="+base64.StdEncoding.EncodeToString([]byte(script)),
|
||||
)
|
||||
var runOut bytes.Buffer
|
||||
runCmd.Stdout = &runOut
|
||||
runCmd.Stderr = &runOut
|
||||
if err := runCmd.Run(); err != nil {
|
||||
return fmt.Errorf("windows winrm bootstrap command failed: %w\n%s", err, runOut.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func windowsBootstrapScript(runnerName string, req LaunchRequest, executor, workdir string) string {
|
||||
if strings.TrimSpace(workdir) == "" {
|
||||
workdir = `C:\burrow\forgejo-runner`
|
||||
}
|
||||
|
||||
runnerExec := strings.TrimSpace(executor)
|
||||
if runnerExec == "" || runnerExec == "shell" {
|
||||
runnerExec = "host"
|
||||
}
|
||||
|
||||
safeName := strings.NewReplacer(`\`, "-", ":", "-", "/", "-", " ", "-").Replace(runnerName)
|
||||
workRoot := strings.TrimRight(workdir, `\`) + `\` + safeName
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("$ErrorActionPreference = 'Stop'\n")
|
||||
b.WriteString("$ProgressPreference = 'SilentlyContinue'\n")
|
||||
b.WriteString("[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12\n")
|
||||
b.WriteString("$runnerName = " + powershellSingleQuote(runnerName) + "\n")
|
||||
b.WriteString("$runnerToken = " + powershellSingleQuote(req.Token) + "\n")
|
||||
b.WriteString("$instanceURL = " + powershellSingleQuote(req.InstanceURL) + "\n")
|
||||
b.WriteString("$labelsCsv = " + powershellSingleQuote(strings.Join(req.Labels, ",")) + "\n")
|
||||
b.WriteString("$runnerExec = " + powershellSingleQuote(runnerExec) + "\n")
|
||||
b.WriteString("$workRoot = " + powershellSingleQuote(workRoot) + "\n")
|
||||
b.WriteString(`
|
||||
New-Item -Path $workRoot -ItemType Directory -Force | Out-Null
|
||||
Set-Location $workRoot
|
||||
|
||||
$runnerVersion = "12.6.4"
|
||||
$zipUrl = "https://code.forgejo.org/forgejo/runner/releases/download/v${runnerVersion}/forgejo-runner-${runnerVersion}-windows-amd64.zip"
|
||||
$zipPath = Join-Path $workRoot "forgejo-runner.zip"
|
||||
$extractDir = Join-Path $workRoot "forgejo-runner"
|
||||
|
||||
if (Test-Path $extractDir) {
|
||||
Remove-Item -Path $extractDir -Recurse -Force
|
||||
}
|
||||
|
||||
Invoke-WebRequest -Uri $zipUrl -OutFile $zipPath
|
||||
Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force
|
||||
|
||||
$runnerExe = Join-Path $extractDir "forgejo-runner.exe"
|
||||
if (-not (Test-Path $runnerExe)) {
|
||||
throw "Missing forgejo-runner.exe after extract: $runnerExe"
|
||||
}
|
||||
|
||||
$labels = @()
|
||||
foreach ($label in ($labelsCsv -split ",")) {
|
||||
$trimmed = $label.Trim()
|
||||
if ([string]::IsNullOrWhiteSpace($trimmed)) { continue }
|
||||
if ($trimmed.Contains(":")) {
|
||||
$labels += $trimmed
|
||||
} else {
|
||||
$labels += ("{0}:{1}" -f $trimmed, $runnerExec)
|
||||
}
|
||||
}
|
||||
if ($labels.Count -eq 0) {
|
||||
throw "No runner labels resolved for windows bootstrap"
|
||||
}
|
||||
|
||||
$labelLines = ($labels | ForEach-Object { " - $_" }) -join [Environment]::NewLine
|
||||
$configPath = Join-Path $workRoot "runner.yaml"
|
||||
$runnerYaml = @"
|
||||
log:
|
||||
level: info
|
||||
runner:
|
||||
file: .runner
|
||||
capacity: 1
|
||||
name: $runnerName
|
||||
labels:
|
||||
$labelLines
|
||||
cache:
|
||||
enabled: false
|
||||
"@
|
||||
Set-Content -Path $configPath -Value $runnerYaml -Encoding UTF8
|
||||
|
||||
$labelsArg = ($labels -join ",")
|
||||
& $runnerExe register --no-interactive --instance $instanceURL --token $runnerToken --name $runnerName --labels $labelsArg --config $configPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw ("forgejo-runner register failed: {0}" -f $LASTEXITCODE)
|
||||
}
|
||||
|
||||
& $runnerExe one-job --config $configPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw ("forgejo-runner one-job failed: {0}" -f $LASTEXITCODE)
|
||||
}
|
||||
`)
|
||||
return b.String()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue