package nsc import ( "bufio" "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "time" ) type windowsProxyOutput struct { Endpoint string `json:"endpoint"` RDP struct { Credentials struct { Username string `json:"username"` Password string `json:"password"` } `json:"credentials"` } `json:"rdp"` } func (d *Dispatcher) launchWindowsRunnerViaWinRM(ctx context.Context, runnerName string, req LaunchRequest, ttl time.Duration, machineType string) error { script := windowsBootstrapScript(runnerName, req, d.opts.Executor, d.opts.WorkDir) return d.launchWindowsScriptViaWinRM(ctx, runnerName, ttl, machineType, req.Labels, script) } func (d *Dispatcher) launchWindowsScriptViaWinRM(ctx context.Context, runnerName string, ttl time.Duration, machineType string, labels []string, script string) error { if ttl <= 0 { ttl = d.opts.DefaultDuration } mt := normalizeWindowsMachineType(machineType, labels) instanceID, createOutput, err := d.createWindowsInstance(ctx, runnerName, ttl, mt) if err != nil { return fmt.Errorf("windows create failed: %w\n%s", err, createOutput) } defer d.destroyNSCInstance(context.Background(), runnerName, instanceID) username, password, err := d.resolveWindowsCredentials(ctx, instanceID) if err != nil { return err } if err := d.probeWindowsWinRMService(ctx, instanceID); err != nil { return err } endpoint, stopForward, err := d.startWindowsWinRMPortForward(ctx, instanceID) if err != nil { return err } defer stopForward() if err := d.runWindowsWinRMPowerShell(ctx, endpoint, username, password, script); err != nil { return err } return nil } func (d *Dispatcher) createWindowsInstance(ctx context.Context, runnerName string, ttl time.Duration, machineType string) (instanceID string, output string, err error) { tmpDir, err := os.MkdirTemp("", "forgejo-nsc-windows-*") if err != nil { return "", "", fmt.Errorf("mktemp: %w", err) } defer os.RemoveAll(tmpDir) metaPath := filepath.Join(tmpDir, "create.json") cidPath := filepath.Join(tmpDir, "create.cid") args := []string{ "create", "--duration", ttl.String(), "--machine_type", machineType, "--cidfile", cidPath, "--purpose", fmt.Sprintf("burrow forgejo runner %s", runnerName), "--output", "plain", "--output_json_to", metaPath, "--wait_timeout", "6m", } args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL) createCtx, cancel := context.WithTimeout(ctx, 8*time.Minute) defer cancel() cmd := exec.CommandContext(createCtx, d.opts.BinaryPath, args...) var buf bytes.Buffer cmd.Stdout = &buf cmd.Stderr = &buf if err := cmd.Run(); err != nil { if created := strings.TrimSpace(mustReadFile(cidPath)); created != "" { d.destroyNSCInstance(context.Background(), runnerName, created) } if errors.Is(createCtx.Err(), context.DeadlineExceeded) { return "", buf.String(), fmt.Errorf("nsc create timed out after %s", 8*time.Minute) } return "", buf.String(), fmt.Errorf("nsc create failed: %w", err) } instanceID, err = readNSCCreateInstanceID(metaPath) if err != nil { return "", buf.String(), fmt.Errorf("nsc create output parse failed: %w", err) } if instanceID == "" { return "", buf.String(), errors.New("nsc create returned empty instance id") } return instanceID, buf.String(), nil } func (d *Dispatcher) resolveWindowsCredentials(ctx context.Context, instanceID string) (username string, password string, err error) { tmpDir, err := os.MkdirTemp("", "forgejo-nsc-winproxy-*") if err != nil { return "", "", fmt.Errorf("mktemp: %w", err) } defer os.RemoveAll(tmpDir) outPath := filepath.Join(tmpDir, "proxy.json") outFile, err := os.Create(outPath) if err != nil { return "", "", fmt.Errorf("create proxy output file: %w", err) } defer outFile.Close() var stderr bytes.Buffer args := []string{"instance", "proxy", instanceID, "-s", "rdp", "-o", "json"} args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL) proxyCtx, cancel := context.WithTimeout(ctx, 90*time.Second) defer cancel() cmd := exec.CommandContext(proxyCtx, d.opts.BinaryPath, args...) cmd.Stdout = outFile cmd.Stderr = &stderr if err := cmd.Start(); err != nil { return "", "", fmt.Errorf("start nsc instance proxy: %w", err) } waitDone := make(chan struct{}) var waitErr error go func() { waitErr = cmd.Wait() close(waitDone) }() var payload windowsProxyOutput deadline := time.Now().Add(45 * time.Second) for time.Now().Before(deadline) { raw, _ := os.ReadFile(outPath) jsonBlob := extractJSON(string(raw)) if jsonBlob != "" { if err := json.Unmarshal([]byte(jsonBlob), &payload); err == nil { username = strings.TrimSpace(payload.RDP.Credentials.Username) password = strings.TrimSpace(payload.RDP.Credentials.Password) if username != "" && password != "" { break } } } select { case <-waitDone: if waitErr != nil { return "", "", fmt.Errorf("nsc instance proxy exited before credentials were available: %w\n%s", waitErr, stderr.String()) } default: } time.Sleep(1 * time.Second) } if cmd.Process != nil { _ = cmd.Process.Kill() } <-waitDone if username == "" || password == "" { raw, _ := os.ReadFile(outPath) return "", "", fmt.Errorf("failed to resolve windows credentials from nsc instance proxy output\nstdout=%s\nstderr=%s", strings.TrimSpace(string(raw)), strings.TrimSpace(stderr.String())) } return username, password, nil } func (d *Dispatcher) probeWindowsWinRMService(ctx context.Context, instanceID string) error { args := []string{"instance", "proxy", instanceID, "-s", "winrm", "-o", "json", "--once"} args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL) probeCtx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() cmd := exec.CommandContext(probeCtx, d.opts.BinaryPath, args...) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out err := cmd.Run() raw := strings.TrimSpace(out.String()) if endpoint, ok := parseProxyEndpoint(raw); ok && endpoint != "" { return nil } if indicatesMissingProxyService(raw, "winrm") { return fmt.Errorf("namespace windows non-interactive channel unavailable: instance does not expose winrm service (rdp-only)\n%s", raw) } if errors.Is(probeCtx.Err(), context.DeadlineExceeded) { return fmt.Errorf("timed out probing Namespace winrm service before bootstrap\n%s", raw) } if err != nil { return fmt.Errorf("nsc winrm service probe failed: %w\n%s", err, raw) } return fmt.Errorf("nsc winrm service probe did not yield endpoint output\n%s", raw) } func parseProxyEndpoint(raw string) (string, bool) { jsonBlob := extractJSON(raw) if jsonBlob == "" { return "", false } var payload struct { Endpoint string `json:"endpoint"` } if err := json.Unmarshal([]byte(jsonBlob), &payload); err != nil { return "", false } endpoint := strings.TrimSpace(payload.Endpoint) if endpoint == "" { return "", false } return endpoint, true } func indicatesMissingProxyService(raw string, service string) bool { service = strings.TrimSpace(service) if service == "" { return false } token := fmt.Sprintf("does not have service %q", service) return strings.Contains(raw, token) } func (d *Dispatcher) startWindowsWinRMPortForward(ctx context.Context, instanceID string) (endpoint string, stop func(), err error) { args := []string{"instance", "port-forward", instanceID, "--target_port", "5985"} args = prependNSCRegionArgs(args, d.opts.ComputeBaseURL) forwardCtx, cancel := context.WithCancel(ctx) cmd := exec.CommandContext(forwardCtx, d.opts.BinaryPath, args...) stdout, err := cmd.StdoutPipe() if err != nil { cancel() return "", nil, fmt.Errorf("port-forward stdout pipe: %w", err) } var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Start(); err != nil { cancel() return "", nil, fmt.Errorf("start nsc port-forward: %w", err) } waitDone := make(chan struct{}) var waitErr error go func() { waitErr = cmd.Wait() close(waitDone) }() endpointCh := make(chan string, 1) scanErrCh := make(chan error, 1) go func() { scanner := bufio.NewScanner(stdout) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if strings.HasPrefix(line, "Listening on ") { endpointCh <- strings.TrimSpace(strings.TrimPrefix(line, "Listening on ")) return } } if err := scanner.Err(); err != nil { scanErrCh <- err } }() select { case endpoint = <-endpointCh: stop = func() { cancel() if cmd.Process != nil { _ = cmd.Process.Kill() } <-waitDone } return endpoint, stop, nil case err := <-scanErrCh: cancel() if cmd.Process != nil { _ = cmd.Process.Kill() } <-waitDone return "", nil, fmt.Errorf("failed reading port-forward output: %w", err) case <-waitDone: cancel() if waitErr != nil { return "", nil, fmt.Errorf("nsc port-forward exited early: %w\n%s", waitErr, stderr.String()) } return "", nil, fmt.Errorf("nsc port-forward exited without endpoint\n%s", stderr.String()) case <-time.After(45 * time.Second): cancel() if cmd.Process != nil { _ = cmd.Process.Kill() } <-waitDone return "", nil, fmt.Errorf("timed out waiting for WinRM port-forward endpoint\n%s", stderr.String()) case <-ctx.Done(): cancel() if cmd.Process != nil { _ = cmd.Process.Kill() } <-waitDone return "", nil, ctx.Err() } } func (d *Dispatcher) runWindowsWinRMPowerShell(ctx context.Context, endpoint, username, password, script string) error { pythonPath, err := exec.LookPath("python3") if err != nil { return fmt.Errorf("python3 is required for windows WinRM bootstrap: %w", err) } workdir := strings.TrimSpace(d.opts.WorkDir) if workdir == "" { workdir = "/tmp/forgejo-runner" } if err := os.MkdirAll(workdir, 0o755); err != nil { return fmt.Errorf("create workdir %s: %w", workdir, err) } venvPath := filepath.Join(workdir, ".winrm-venv") venvPython := filepath.Join(venvPath, "bin", "python") if _, err := os.Stat(venvPython); err != nil { cmd := exec.CommandContext(ctx, pythonPath, "-m", "venv", venvPath) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := cmd.Run(); err != nil { return fmt.Errorf("create python venv for winrm failed: %w\n%s", err, out.String()) } } ensurePyWinRM := ` import importlib.util, subprocess, sys if importlib.util.find_spec("winrm") is None: subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", "pywinrm"]) ` ensureCmd := exec.CommandContext(ctx, venvPython, "-c", ensurePyWinRM) var ensureOut bytes.Buffer ensureCmd.Stdout = &ensureOut ensureCmd.Stderr = &ensureOut if err := ensureCmd.Run(); err != nil { return fmt.Errorf("install pywinrm failed: %w\n%s", err, ensureOut.String()) } runScript := ` import base64, os, sys, time, traceback, winrm endpoint = os.environ["WINRM_ENDPOINT"] user = os.environ["WINRM_USER"] password = os.environ["WINRM_PASS"] script = base64.b64decode(os.environ["WINRM_SCRIPT_B64"]).decode("utf-8") deadline = time.time() + 300.0 last_err = None while time.time() < deadline: try: session = winrm.Session(f"http://{endpoint}/wsman", auth=(user, password), transport="ntlm") result = session.run_ps(script) sys.stdout.write(result.std_out.decode("utf-8", errors="replace")) sys.stderr.write(result.std_err.decode("utf-8", errors="replace")) print(f"winrm_exit={result.status_code}") sys.exit(result.status_code) except Exception as err: last_err = err time.sleep(5.0) sys.stderr.write("timed out waiting for WinRM connectivity after 300s\\n") if last_err is not None: traceback.print_exception(last_err, file=sys.stderr) sys.exit(111) ` runCmd := exec.CommandContext(ctx, venvPython, "-c", runScript) runCmd.Env = append(os.Environ(), "WINRM_ENDPOINT="+endpoint, "WINRM_USER="+username, "WINRM_PASS="+password, "WINRM_SCRIPT_B64="+base64.StdEncoding.EncodeToString([]byte(script)), ) var runOut bytes.Buffer runCmd.Stdout = &runOut runCmd.Stderr = &runOut if err := runCmd.Run(); err != nil { return fmt.Errorf("windows winrm bootstrap command failed: %w\n%s", err, runOut.String()) } return nil } func windowsBootstrapScript(runnerName string, req LaunchRequest, executor, workdir string) string { if strings.TrimSpace(workdir) == "" { workdir = `C:\burrow\forgejo-runner` } runnerExec := strings.TrimSpace(executor) if runnerExec == "" || runnerExec == "shell" { runnerExec = "host" } safeName := strings.NewReplacer(`\`, "-", ":", "-", "/", "-", " ", "-").Replace(runnerName) workRoot := strings.TrimRight(workdir, `\`) + `\` + safeName var b strings.Builder b.WriteString("$ErrorActionPreference = 'Stop'\n") b.WriteString("$ProgressPreference = 'SilentlyContinue'\n") b.WriteString("[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12\n") b.WriteString("$runnerName = " + powershellSingleQuote(runnerName) + "\n") b.WriteString("$runnerToken = " + powershellSingleQuote(req.Token) + "\n") b.WriteString("$instanceURL = " + powershellSingleQuote(req.InstanceURL) + "\n") b.WriteString("$labelsCsv = " + powershellSingleQuote(strings.Join(req.Labels, ",")) + "\n") b.WriteString("$runnerExec = " + powershellSingleQuote(runnerExec) + "\n") b.WriteString("$workRoot = " + powershellSingleQuote(workRoot) + "\n") b.WriteString(` New-Item -Path $workRoot -ItemType Directory -Force | Out-Null Set-Location $workRoot $runnerVersion = "12.6.4" $zipUrl = "https://code.forgejo.org/forgejo/runner/releases/download/v${runnerVersion}/forgejo-runner-${runnerVersion}-windows-amd64.zip" $zipPath = Join-Path $workRoot "forgejo-runner.zip" $extractDir = Join-Path $workRoot "forgejo-runner" if (Test-Path $extractDir) { Remove-Item -Path $extractDir -Recurse -Force } Invoke-WebRequest -Uri $zipUrl -OutFile $zipPath Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force $runnerExe = Join-Path $extractDir "forgejo-runner.exe" if (-not (Test-Path $runnerExe)) { throw "Missing forgejo-runner.exe after extract: $runnerExe" } $labels = @() foreach ($label in ($labelsCsv -split ",")) { $trimmed = $label.Trim() if ([string]::IsNullOrWhiteSpace($trimmed)) { continue } if ($trimmed.Contains(":")) { $labels += $trimmed } else { $labels += ("{0}:{1}" -f $trimmed, $runnerExec) } } if ($labels.Count -eq 0) { throw "No runner labels resolved for windows bootstrap" } $labelLines = ($labels | ForEach-Object { " - $_" }) -join [Environment]::NewLine $configPath = Join-Path $workRoot "runner.yaml" $runnerYaml = @" log: level: info runner: file: .runner capacity: 1 name: $runnerName labels: $labelLines cache: enabled: false "@ Set-Content -Path $configPath -Value $runnerYaml -Encoding UTF8 $labelsArg = ($labels -join ",") & $runnerExe register --no-interactive --instance $instanceURL --token $runnerToken --name $runnerName --labels $labelsArg --config $configPath if ($LASTEXITCODE -ne 0) { throw ("forgejo-runner register failed: {0}" -f $LASTEXITCODE) } & $runnerExe one-job --config $configPath if ($LASTEXITCODE -ne 0) { throw ("forgejo-runner one-job failed: {0}" -f $LASTEXITCODE) } `) return b.String() }