remote controller working!

also continue fix attempt on sheepit not starting

remote_worker_attach.ps1

@@ -88,7 +88,16 @@ try {
             continue
         }
 
-        if ($input.Trim().ToLower() -eq 'detach') {
+        $normalized = $input.Trim()
+        if ($normalized.StartsWith(':')) {
+            $normalized = $normalized.TrimStart(':').Trim()
+        }
+
+        if ($normalized.Length -eq 0) {
+            continue
+        }
+
+        if ($normalized.ToLowerInvariant() -eq 'detach') {
             break
         }
 
@@ -97,9 +106,9 @@ try {
 }
 finally {
     if ($logJob) {
-        Stop-Job -Job $logJob -Force -ErrorAction SilentlyContinue
+        Stop-Job -Job $logJob -ErrorAction SilentlyContinue
         Receive-Job -Job $logJob -ErrorAction SilentlyContinue | Out-Null
-        Remove-Job -Job $logJob -Force -ErrorAction SilentlyContinue
+        Remove-Job -Job $logJob -ErrorAction SilentlyContinue
     }
 
     Write-Host "Detached from worker $WorkerName." -ForegroundColor Cyan

remote_worker_controller.ps1

@@ -5,9 +5,10 @@ param(
     [Parameter(Mandatory = $true)]
     [string]$WorkerType,
 
-    [Parameter(Mandatory = $true)]
     [string]$PayloadBase64,
 
+    [string]$PayloadBase64Path,
+
 [string]$DataRoot = (Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'),
 
     [int]$MaxRestarts = 5,
@@ -16,6 +17,7 @@ param(
 )
 
 $ErrorActionPreference = 'Stop'
+[System.Management.Automation.Runspaces.Runspace]::DefaultRunspace = $Host.Runspace
 
 # region Path setup
 $workerRoot = Join-Path -Path $DataRoot -ChildPath $WorkerType
@@ -30,14 +32,124 @@ New-Item -ItemType Directory -Path $stateRoot -Force | Out-Null
 $logPath = Join-Path -Path $logsRoot -ChildPath 'worker.log'
 $metaPath = Join-Path -Path $stateRoot -ChildPath 'worker-info.json'
 $commandPath = Join-Path -Path $stateRoot -ChildPath 'commands.txt'
-$payloadPath = Join-Path -Path $stateRoot -ChildPath "payload-$([Guid]::NewGuid().ToString()).ps1"
+$payloadPath = Join-Path -Path $stateRoot -ChildPath "payload.ps1"
+# endregion
+
+# region Logging
+$logStream = [System.IO.FileStream]::new(
+    $logPath,
+    [System.IO.FileMode]::Append,
+    [System.IO.FileAccess]::Write,
+    [System.IO.FileShare]::ReadWrite
+)
+$logWriter = [System.IO.StreamWriter]::new($logStream, [System.Text.Encoding]::UTF8)
+$logWriter.AutoFlush = $true
+
+# Create C# event handler class that doesn't require PowerShell runspace
+if (-not ("UnifiedWorkers.ProcessLogHandler" -as [type])) {
+    $csharpCode = @'
+using System;
+using System.Diagnostics;
+using System.IO;
+
+namespace UnifiedWorkers
+{
+    public sealed class ProcessLogHandler
+    {
+        private readonly TextWriter _writer;
+        private readonly string _prefix;
+        private readonly object _lock = new object();
+
+        public ProcessLogHandler(TextWriter writer, string prefix)
+        {
+            _writer = writer ?? throw new ArgumentNullException("writer");
+            _prefix = prefix ?? throw new ArgumentNullException("prefix");
+        }
+
+        public void OnDataReceived(object sender, DataReceivedEventArgs e)
+        {
+            if (e == null || string.IsNullOrEmpty(e.Data))
+            {
+                return;
+            }
+
+            lock (_lock)
+            {
+                try
+                {
+                    var timestamp = DateTime.UtcNow.ToString("u");
+                    _writer.WriteLine(string.Format("[{0} {1}] {2}", _prefix, timestamp, e.Data));
+                    _writer.Flush();
+                }
+                catch
+                {
+                    // Ignore write errors to prevent cascading failures
+                }
+            }
+        }
+    }
+}
+'@
+    Add-Type -TypeDefinition $csharpCode -ErrorAction Stop
+}
+
+function Write-LogLine {
+    param(
+        [string]$Prefix,
+        [string]$Message
+    )
+
+    if (-not $logWriter) { return }
+    $timestamp = (Get-Date).ToString('u')
+    $logWriter.WriteLine("[$Prefix $timestamp] $Message")
+}
+
+function Write-ControllerLog {
+    param([string]$Message)
+    Write-LogLine -Prefix 'CTRL' -Message $Message
+}
+
+function Write-FatalLog {
+    param([string]$Message)
+
+    try {
+        Write-ControllerLog $Message
+    }
+    catch {
+        $timestamp = (Get-Date).ToString('u')
+        $fallback = "[CTRL $timestamp] $Message"
+        try {
+            [System.IO.File]::AppendAllText($logPath, $fallback + [Environment]::NewLine, [System.Text.Encoding]::UTF8)
+        }
+        catch {
+            # last resort: write to host
+            Write-Error $fallback
+        }
+    }
+}
 # endregion
 
 # region Helpers
-function Write-ControllerLog {
-    param([string]$Message)
-    $timestamp = (Get-Date).ToString('u')
-    Add-Content -Path $logPath -Value "[CTRL $timestamp] $Message" -Encoding UTF8
+
+function Resolve-PayloadBase64 {
+    if ($PayloadBase64) {
+        return $PayloadBase64.Trim()
+    }
+
+    if ($PayloadBase64Path) {
+        if (-not (Test-Path $PayloadBase64Path)) {
+            throw "Payload file '$PayloadBase64Path' not found."
+        }
+
+        $content = Get-Content -Path $PayloadBase64Path -Raw
+        if ([string]::IsNullOrWhiteSpace($content)) {
+            throw "Payload file '$PayloadBase64Path' is empty."
+        }
+        
+        return $content.Trim()
+    }
+
+    throw "No payload data provided to controller."
 }
 
 function Write-Metadata {
@@ -83,20 +195,36 @@ function Get-PendingCommands {
 # endregion
 
 try {
+    # record initial state before launching worker
+    Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0
+
+    $resolvedPayloadBase64 = Resolve-PayloadBase64
+    $PayloadBase64 = $resolvedPayloadBase64
+
+    try {
         # Write payload script to disk
-    $payloadBytes = [Convert]::FromBase64String($PayloadBase64)
-    [IO.File]::WriteAllBytes($payloadPath, $payloadBytes)
-    Write-ControllerLog "Payload written to $payloadPath"
-}
-catch {
-    Write-Error "Unable to write payload: $($_.Exception.Message)"
-    exit 1
-}
+        # The payload is base64-encoded UTF-16 (Unicode), so decode it properly
+        Write-ControllerLog "Decoding payload base64 (length: $($resolvedPayloadBase64.Length))"
+        $payloadBytes = [Convert]::FromBase64String($resolvedPayloadBase64)
+        Write-ControllerLog "Decoded payload to $($payloadBytes.Length) bytes"
         
-$restartCount = 0
-$controllerPid = $PID
+        # Convert UTF-16 bytes back to string, then write as UTF-8 (PowerShell's preferred encoding)
+        $payloadText = [Text.Encoding]::Unicode.GetString($payloadBytes)
+        [IO.File]::WriteAllText($payloadPath, $payloadText, [Text.Encoding]::UTF8)
+        Write-ControllerLog "Payload written to $payloadPath ($($payloadText.Length) characters)"
+    }
+    catch {
+        Write-FatalLog "Unable to write payload: $($_.Exception.Message)"
+        if ($_.Exception.InnerException) {
+            Write-FatalLog "Inner exception: $($_.Exception.InnerException.Message)"
+        }
+        throw
+    }
 
-while ($restartCount -le $MaxRestarts) {
+    $restartCount = 0
+    $controllerPid = $PID
+
+    while ($restartCount -le $MaxRestarts) {
         try {
             # Initialize worker process
             $psi = [System.Diagnostics.ProcessStartInfo]::new()
@@ -125,23 +253,11 @@ while ($restartCount -le $MaxRestarts) {
             Write-ControllerLog "Worker process started with PID $($workerProcess.Id)"
             Write-Metadata -Status 'running' -WorkerPid $workerProcess.Id -ControllerPid $controllerPid -Restarts $restartCount
 
-        $logWriter = [System.IO.StreamWriter]::new($logPath, $true, [System.Text.Encoding]::UTF8)
-        $logWriter.AutoFlush = $true
+            $stdoutHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'OUT')
+            $stderrHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'ERR')
             
-        $outputHandler = [System.Diagnostics.DataReceivedEventHandler]{
-            param($s, $e)
-            if ($e.Data) {
-                $ts = (Get-Date).ToString('u')
-                $logWriter.WriteLine("[OUT $ts] $($e.Data)")
-            }
-        }
-        $errorHandler = [System.Diagnostics.DataReceivedEventHandler]{
-            param($s, $e)
-            if ($e.Data) {
-                $ts = (Get-Date).ToString('u')
-                $logWriter.WriteLine("[ERR $ts] $($e.Data)")
-            }
-        }
+            $outputHandler = [System.Diagnostics.DataReceivedEventHandler]$stdoutHandler.OnDataReceived
+            $errorHandler = [System.Diagnostics.DataReceivedEventHandler]$stderrHandler.OnDataReceived
 
             $workerProcess.add_OutputDataReceived($outputHandler)
             $workerProcess.add_ErrorDataReceived($errorHandler)
@@ -191,8 +307,24 @@ while ($restartCount -le $MaxRestarts) {
 
         Write-ControllerLog "Restarting worker in $RestartDelaySeconds seconds (attempt $restartCount of $MaxRestarts)."
         Start-Sleep -Seconds $RestartDelaySeconds
+    }
+
+    Write-Metadata -Status 'inactive' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount
+    Write-ControllerLog "Controller exiting."
+}
+catch {
+    Write-FatalLog "Fatal controller error: $($_.Exception.Message)"
+    if ($_.ScriptStackTrace) {
+        Write-FatalLog "Stack: $($_.ScriptStackTrace)"
+    }
+    throw
+}
+finally {
+    if ($logWriter) {
+        $logWriter.Dispose()
+    }
+    if ($logStream) {
+        $logStream.Dispose()
+    }
 }
 
-Write-Metadata -Status 'inactive' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount
-Write-ControllerLog "Controller exiting."
-

unified_sheepit_launcher.ps1

@@ -36,6 +36,53 @@ $script:ControllerScriptBase64 = $null
 $script:AttachHelperScriptBase64 = $null
 $script:WorkerBasePathCache = @{}
 
+function Remove-ClixmlNoise {
+    param([object[]]$Lines)
+
+    $noisePatterns = @(
+        '^#<\s*CLIXML',
+        '^\s*<Objs\b', '^\s*</Objs>',
+        '^\s*<Obj\b', '^\s*</Obj>',
+        '^\s*<TN\b', '^\s*</TN>',
+        '^\s*<MS\b', '^\s*</MS>',
+        '^\s*<PR\b', '^\s*</PR>',
+        '^\s*<I64\b', '^\s*</I64>',
+        '^\s*<AI\b', '^\s*</AI>',
+        '^\s*<Nil\b', '^\s*</Nil>',
+        '^\s*<PI\b', '^\s*</PI>',
+        '^\s*<PC\b', '^\s*</PC>',
+        '^\s*<SR\b', '^\s*</SR>',
+        '^\s*<SD\b', '^\s*</SD>',
+        '^\s*<S\b', '^\s*</S>'
+    )
+
+    $filtered = @()
+    foreach ($entry in $Lines) {
+        if ($null -eq $entry) { continue }
+        $text = $entry.ToString()
+        $isNoise = $false
+        foreach ($pattern in $noisePatterns) {
+            if ($text -match $pattern) {
+                $isNoise = $true
+                break
+            }
+        }
+
+        if (-not $isNoise) {
+            $filtered += $text
+        }
+    }
+    return $filtered
+}
+
+function Write-FilteredSshOutput {
+    param([object[]]$Lines)
+    $clean = Remove-ClixmlNoise -Lines $Lines
+    foreach ($line in $clean) {
+        Write-Host $line
+    }
+}
+
 function Build-SshArgsFromParts {
     param(
         [pscustomobject]$Parts,
@@ -102,8 +149,9 @@ function Get-WorkerBasePath {
     $sshArgs = Build-SshArgsFromParts -Parts $ConnectionParts -Interactive:$false
     $scriptBlock = "`$ProgressPreference='SilentlyContinue'; [Environment]::GetFolderPath('LocalApplicationData')"
     $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($scriptBlock))
-    $remoteCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand $encoded"
-    $output = & ssh @sshArgs $remoteCmd
+    $remoteCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand $encoded"
+    $rawOutput = & ssh @sshArgs $remoteCmd 2>&1
+    $output = Remove-ClixmlNoise -Lines $rawOutput
     if ($LASTEXITCODE -ne 0) {
         throw "Unable to determine LocalAppData on $($Worker.Name)."
     }
@@ -230,11 +278,13 @@ function Invoke-RemotePowerShell {
     $remoteTarget = "{0}:{1}" -f $parts.Host, ('"'+$remoteScriptScp+'"')
 
     $ensureScript = "New-Item -ItemType Directory -Path '$remoteTmpDir' -Force | Out-Null"
-    $ensureCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($ensureScript))
-    & ssh @sshBaseArgs $ensureCmd
-    if ($LASTEXITCODE -ne 0) {
+    $ensureCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($ensureScript))
+    $ensureOutput = & ssh @sshBaseArgs $ensureCmd 2>&1
+    $ensureExit = $LASTEXITCODE
+    Write-FilteredSshOutput -Lines $ensureOutput
+    if ($ensureExit -ne 0) {
         Remove-Item $localTemp -ErrorAction SilentlyContinue
-        return $LASTEXITCODE
+        return $ensureExit
     }
 
     $scpArgs = Build-ScpArgsFromParts -Parts $parts
@@ -248,15 +298,47 @@ function Invoke-RemotePowerShell {
         return $scpExit
     }
 
-    $execCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$remoteScriptWin`""
-    & ssh @sshBaseArgs $execCmd
+    $execCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -File `"$remoteScriptWin`""
+    $execOutput = & ssh @sshBaseArgs $execCmd 2>&1
     $execExit = $LASTEXITCODE
+    Write-FilteredSshOutput -Lines $execOutput
 
     $cleanupScript = "Remove-Item -LiteralPath '$remoteScriptWin' -ErrorAction SilentlyContinue"
-    $cleanupCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($cleanupScript))
-    & ssh @sshBaseArgs $cleanupCmd | Out-Null
+    $cleanupCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($cleanupScript))
+    $cleanupOutput = & ssh @sshBaseArgs $cleanupCmd 2>&1
+    Write-FilteredSshOutput -Lines $cleanupOutput
 
-    return $execExit
+    return [int]$execExit
+}
+
+function Resolve-ExitCode {
+    param($Value)
+
+    if ($Value -is [System.Array]) {
+        for ($i = $Value.Count - 1; $i -ge 0; $i--) {
+            $candidate = Resolve-ExitCode -Value $Value[$i]
+            if ($candidate -ne $null) {
+                return $candidate
+            }
+        }
+        return 0
+    }
+
+    if ($Value -is [int]) {
+        return $Value
+    }
+
+    $text = $Value
+    if ($null -eq $text) {
+        return 0
+    }
+
+    $parsed = 0
+    if ([int]::TryParse($text.ToString(), [ref]$parsed)) {
+        return $parsed
+    }
+
+    return 0
 }
 
 function Ensure-ControllerDeployed {
@@ -269,7 +351,10 @@ New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
 `$controllerPath = Join-Path `$dataRoot 'controller.ps1'
 [IO.File]::WriteAllBytes(`$controllerPath, [Convert]::FromBase64String('$controllerBase64'))
 "@
-    Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
+    $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $script)
+    if ($exit -ne 0) {
+        throw "Controller deployment failed on $($Worker.Name) (exit $exit)."
+    }
 }
 
 function Ensure-AttachHelperDeployed {
@@ -282,7 +367,10 @@ New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
 `$attachPath = Join-Path `$dataRoot 'attach-helper.ps1'
 [IO.File]::WriteAllBytes(`$attachPath, [Convert]::FromBase64String('$helperBase64'))
 "@
-    Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
+    $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $script)
+    if ($exit -ne 0) {
+        throw "Attach helper deployment failed on $($Worker.Name) (exit $exit)."
+    }
 }
 
 function Get-EnsureWorkerScript {
@@ -308,6 +396,17 @@ function Get-EnsureWorkerScript {
 `$payloadBase64 = `$params.PayloadBase64
 `$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
 `$instanceRoot = Join-Path (Join-Path `$dataRoot `$workerType) `$workerName
+`$logsRoot = Join-Path `$instanceRoot 'logs'
+`$stateRoot = Join-Path `$instanceRoot 'state'
+New-Item -ItemType Directory -Path `$logsRoot -Force | Out-Null
+New-Item -ItemType Directory -Path `$stateRoot -Force | Out-Null
+`$logPath = Join-Path `$logsRoot 'worker.log'
+`$commandPath = Join-Path `$stateRoot 'commands.txt'
+`$payloadPath = Join-Path `$stateRoot 'payload.ps1'
+`$payloadBase64Path = Join-Path `$stateRoot 'payload.b64'
+if (-not (Test-Path `$logPath)) { New-Item -Path `$logPath -ItemType File -Force | Out-Null }
+if (-not (Test-Path `$commandPath)) { New-Item -Path `$commandPath -ItemType File -Force | Out-Null }
+[IO.File]::WriteAllText(`$payloadBase64Path, `$payloadBase64, [System.Text.Encoding]::UTF8)
 `$metaPath = Join-Path `$instanceRoot 'state\worker-info.json'
 `$controllerPath = Join-Path `$dataRoot 'controller.ps1'
 
@@ -331,6 +430,21 @@ if (Test-Path `$metaPath) {
 }
 
 if (`$shouldStart) {
+    `$initialMeta = [pscustomobject]@{
+        WorkerName    = `$workerName
+        WorkerType    = `$workerType
+        Status        = 'launching'
+        ControllerPid = `$null
+        WorkerPid     = `$null
+        Restarts      = 0
+        LastExitCode  = `$null
+        LogPath       = `$logPath
+        CommandPath   = `$commandPath
+        PayloadPath   = `$payloadPath
+        UpdatedAtUtc  = (Get-Date).ToUniversalTime()
+    } | ConvertTo-Json -Depth 5
+    `$initialMeta | Set-Content -Path `$metaPath -Encoding UTF8
+
     `$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
     if (`$pwsh) {
         `$psExe = `$pwsh.Source
@@ -344,7 +458,7 @@ if (`$shouldStart) {
         '-File',"`$controllerPath",
         '-WorkerName',"`$workerName",
         '-WorkerType',"`$workerType",
-        '-PayloadBase64',"`$payloadBase64"
+        '-PayloadBase64Path',"`$payloadBase64Path"
     )
 
     Start-Process -FilePath `$psExe -ArgumentList `$controllerArgs -WindowStyle Hidden | Out-Null
@@ -363,7 +477,10 @@ function Ensure-PersistentWorker {
     Ensure-ControllerDeployed -Worker $Worker
     $payloadBase64 = ConvertTo-Base64Unicode -Content $PayloadScript
     $ensureScript = Get-EnsureWorkerScript -WorkerName $Worker.Name -WorkerType $WorkerType -PayloadBase64 $payloadBase64
-    Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript | Out-Null
+    $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript)
+    if ($exit -ne 0) {
+        throw "Worker ensure script failed on $($Worker.Name) (exit $exit)."
+    }
 }
 
 function Test-WorkerMetadataExists {
@@ -561,11 +678,21 @@ function Ensure-SheepItWorkerController {
 
 function Start-SheepItWorker {
     param([object]$Worker)
+    try {
+        Write-Host "Ensuring SheepIt controller on $($Worker.Name)..." -ForegroundColor Cyan
         Ensure-SheepItWorkerController -Worker $Worker
-    if (-not (Wait-WorkerMetadata -Worker $Worker -WorkerType 'sheepit' -TimeoutSeconds 20)) {
-        Write-Host "Worker metadata did not appear on $($Worker.Name). Check controller logs." -ForegroundColor Red
+    }
+    catch {
+        Write-Host "Failed to ensure controller on $($Worker.Name): $($_.Exception.Message)" -ForegroundColor Red
         return
     }
+
+    if (-not (Wait-WorkerMetadata -Worker $Worker -WorkerType 'sheepit' -TimeoutSeconds 30)) {
+        Write-Host "Worker metadata did not appear on $($Worker.Name). Check controller logs under %LocalAppData%\UnifiedWorkers." -ForegroundColor Red
+        return
+    }
+
+    Write-Host "Controller ready. Attaching to SheepIt worker on $($Worker.Name)..." -ForegroundColor Cyan
     Invoke-WorkerAttach -Worker $Worker -WorkerType 'sheepit'
 }
 
@@ -643,7 +770,11 @@ while ($true) {
     Write-Host "5. Quit all workers" -ForegroundColor Yellow
     Write-Host "6. Exit" -ForegroundColor Yellow
 
-    $choice = Read-Host "Select option (1-3)"
+    $choice = Read-Host "Select option (1-6)"
+
+    if ($choice -eq '6') {
+        break
+    }
 
     switch ($choice) {
         '1' { Select-SheepItWorker }
@@ -651,7 +782,6 @@ while ($true) {
         '3' { Send-SheepItCommandAll -CommandText 'pause' }
         '4' { Send-SheepItCommandAll -CommandText 'resume' }
         '5' { Send-SheepItCommandAll -CommandText 'quit' }
-        '6' { break }
         default {
             Write-Host "Invalid selection." -ForegroundColor Red
             Start-Sleep -Seconds 1