remote controller working!

This commit is contained in:
Nathan
2025-12-02 16:23:31 -07:00
parent e568e02cdf
commit 428d9bac6b
2 changed files with 2560 additions and 135 deletions

File diff suppressed because one or more lines are too long

View File

@@ -45,50 +45,52 @@ $logStream = [System.IO.FileStream]::new(
$logWriter = [System.IO.StreamWriter]::new($logStream, [System.Text.Encoding]::UTF8) $logWriter = [System.IO.StreamWriter]::new($logStream, [System.Text.Encoding]::UTF8)
$logWriter.AutoFlush = $true $logWriter.AutoFlush = $true
if (-not ("UnifiedWorkers.WorkerLogSink" -as [type])) { # Create C# event handler class that doesn't require PowerShell runspace
Add-Type -Namespace UnifiedWorkers -Name WorkerLogSink -MemberDefinition @" if (-not ("UnifiedWorkers.ProcessLogHandler" -as [type])) {
$csharpCode = @'
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
public sealed class WorkerLogSink namespace UnifiedWorkers
{ {
private readonly TextWriter _writer; public sealed class ProcessLogHandler
private readonly string _prefix;
private readonly object _sync = new object();
public WorkerLogSink(TextWriter writer, string prefix)
{ {
if (writer == null) private readonly TextWriter _writer;
private readonly string _prefix;
private readonly object _lock = new object();
public ProcessLogHandler(TextWriter writer, string prefix)
{ {
throw new ArgumentNullException(\"writer\"); _writer = writer ?? throw new ArgumentNullException("writer");
} _prefix = prefix ?? throw new ArgumentNullException("prefix");
if (prefix == null)
{
throw new ArgumentNullException(\"prefix\");
} }
_writer = writer; public void OnDataReceived(object sender, DataReceivedEventArgs e)
_prefix = prefix;
}
public void OnData(object sender, DataReceivedEventArgs e)
{
if (e == null || string.IsNullOrEmpty(e.Data))
{ {
return; if (e == null || string.IsNullOrEmpty(e.Data))
} {
return;
}
lock (_sync) lock (_lock)
{ {
var timestamp = DateTime.UtcNow.ToString("u"); try
var line = string.Format("[{0} {1}] {2}", _prefix, timestamp, e.Data); {
_writer.WriteLine(line); var timestamp = DateTime.UtcNow.ToString("u");
_writer.Flush(); _writer.WriteLine(string.Format("[{0} {1}] {2}", _prefix, timestamp, e.Data));
_writer.Flush();
}
catch
{
// Ignore write errors to prevent cascading failures
}
}
} }
} }
} }
"@ '@
Add-Type -TypeDefinition $csharpCode -ErrorAction Stop
} }
function Write-LogLine { function Write-LogLine {
@@ -99,20 +101,39 @@ function Write-LogLine {
if (-not $logWriter) { return } if (-not $logWriter) { return }
$timestamp = (Get-Date).ToString('u') $timestamp = (Get-Date).ToString('u')
$logWriter.WriteLine("[{0} {1}] {2}" -f $Prefix, $timestamp, $Message) $logWriter.WriteLine("[$Prefix $timestamp] $Message")
} }
function Write-ControllerLog { function Write-ControllerLog {
param([string]$Message) param([string]$Message)
Write-LogLine -Prefix 'CTRL' -Message $Message Write-LogLine -Prefix 'CTRL' -Message $Message
} }
function Write-FatalLog {
param([string]$Message)
try {
Write-ControllerLog $Message
}
catch {
$timestamp = (Get-Date).ToString('u')
$fallback = "[CTRL $timestamp] $Message"
try {
[System.IO.File]::AppendAllText($logPath, $fallback + [Environment]::NewLine, [System.Text.Encoding]::UTF8)
}
catch {
# last resort: write to host
Write-Error $fallback
}
}
}
# endregion # endregion
# region Helpers # region Helpers
function Resolve-PayloadBase64 { function Resolve-PayloadBase64 {
if ($PayloadBase64) { if ($PayloadBase64) {
return $PayloadBase64 return $PayloadBase64.Trim()
} }
if ($PayloadBase64Path) { if ($PayloadBase64Path) {
@@ -120,7 +141,12 @@ function Resolve-PayloadBase64 {
throw "Payload file '$PayloadBase64Path' not found." throw "Payload file '$PayloadBase64Path' not found."
} }
return (Get-Content -Path $PayloadBase64Path -Raw) $content = Get-Content -Path $PayloadBase64Path -Raw
if ([string]::IsNullOrWhiteSpace($content)) {
throw "Payload file '$PayloadBase64Path' is empty."
}
return $content.Trim()
} }
throw "No payload data provided to controller." throw "No payload data provided to controller."
@@ -168,117 +194,137 @@ function Get-PendingCommands {
} }
# endregion # endregion
# record initial state before launching worker
Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0
$resolvedPayloadBase64 = Resolve-PayloadBase64
$PayloadBase64 = $resolvedPayloadBase64
try { try {
# Write payload script to disk # record initial state before launching worker
$payloadBytes = [Convert]::FromBase64String($resolvedPayloadBase64) Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0
[IO.File]::WriteAllBytes($payloadPath, $payloadBytes)
Write-ControllerLog "Payload written to $payloadPath"
}
catch {
Write-Error "Unable to write payload: $($_.Exception.Message)"
exit 1
}
$restartCount = 0 $resolvedPayloadBase64 = Resolve-PayloadBase64
$controllerPid = $PID $PayloadBase64 = $resolvedPayloadBase64
while ($restartCount -le $MaxRestarts) {
try { try {
# Initialize worker process # Write payload script to disk
$psi = [System.Diagnostics.ProcessStartInfo]::new() # The payload is base64-encoded UTF-16 (Unicode), so decode it properly
$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue Write-ControllerLog "Decoding payload base64 (length: $($resolvedPayloadBase64.Length))"
if ($pwsh) { $payloadBytes = [Convert]::FromBase64String($resolvedPayloadBase64)
$psi.FileName = $pwsh.Source Write-ControllerLog "Decoded payload to $($payloadBytes.Length) bytes"
}
else {
$psi.FileName = (Get-Command powershell -ErrorAction Stop).Source
}
$psi.Arguments = "-NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$payloadPath`"" # Convert UTF-16 bytes back to string, then write as UTF-8 (PowerShell's preferred encoding)
$psi.UseShellExecute = $false $payloadText = [Text.Encoding]::Unicode.GetString($payloadBytes)
$psi.RedirectStandardInput = $true [IO.File]::WriteAllText($payloadPath, $payloadText, [Text.Encoding]::UTF8)
$psi.RedirectStandardOutput = $true Write-ControllerLog "Payload written to $payloadPath ($($payloadText.Length) characters)"
$psi.RedirectStandardError = $true
$psi.CreateNoWindow = $true
$workerProcess = New-Object System.Diagnostics.Process
$workerProcess.StartInfo = $psi
if (-not $workerProcess.Start()) {
throw "Failed to start worker process."
}
Write-ControllerLog "Worker process started with PID $($workerProcess.Id)"
Write-Metadata -Status 'running' -WorkerPid $workerProcess.Id -ControllerPid $controllerPid -Restarts $restartCount
$stdoutSink = [UnifiedWorkers.WorkerLogSink]::new($logWriter, 'OUT')
$stderrSink = [UnifiedWorkers.WorkerLogSink]::new($logWriter, 'ERR')
$outputHandler = [System.Diagnostics.DataReceivedEventHandler]$stdoutSink.OnData
$errorHandler = [System.Diagnostics.DataReceivedEventHandler]$stderrSink.OnData
$workerProcess.add_OutputDataReceived($outputHandler)
$workerProcess.add_ErrorDataReceived($errorHandler)
$workerProcess.BeginOutputReadLine()
$workerProcess.BeginErrorReadLine()
while (-not $workerProcess.HasExited) {
$commands = Get-PendingCommands
foreach ($command in $commands) {
$trimmed = $command.Trim()
if (-not $trimmed) { continue }
Write-ControllerLog "Received command '$trimmed'"
try {
$workerProcess.StandardInput.WriteLine($trimmed)
$workerProcess.StandardInput.Flush()
}
catch {
Write-ControllerLog "Failed to forward command '$trimmed': $($_.Exception.Message)"
}
if ($trimmed -ieq 'quit') {
Write-ControllerLog "Quit command issued. Waiting for worker to exit."
}
}
Start-Sleep -Milliseconds 500
}
$exitCode = $workerProcess.ExitCode
Write-ControllerLog "Worker exited with code $exitCode"
Write-Metadata -Status 'stopped' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount -LastExitCode $exitCode
if ($exitCode -eq 0) {
break
}
} }
catch { catch {
Write-ControllerLog "Controller error: $($_.Exception.Message)" Write-FatalLog "Unable to write payload: $($_.Exception.Message)"
if ($_.Exception.InnerException) {
Write-FatalLog "Inner exception: $($_.Exception.InnerException.Message)"
}
throw
} }
$restartCount++ $restartCount = 0
if ($restartCount -gt $MaxRestarts) { $controllerPid = $PID
Write-ControllerLog "Maximum restart attempts reached. Controller stopping."
break while ($restartCount -le $MaxRestarts) {
try {
# Initialize worker process
$psi = [System.Diagnostics.ProcessStartInfo]::new()
$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
if ($pwsh) {
$psi.FileName = $pwsh.Source
}
else {
$psi.FileName = (Get-Command powershell -ErrorAction Stop).Source
}
$psi.Arguments = "-NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$payloadPath`""
$psi.UseShellExecute = $false
$psi.RedirectStandardInput = $true
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.CreateNoWindow = $true
$workerProcess = New-Object System.Diagnostics.Process
$workerProcess.StartInfo = $psi
if (-not $workerProcess.Start()) {
throw "Failed to start worker process."
}
Write-ControllerLog "Worker process started with PID $($workerProcess.Id)"
Write-Metadata -Status 'running' -WorkerPid $workerProcess.Id -ControllerPid $controllerPid -Restarts $restartCount
$stdoutHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'OUT')
$stderrHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'ERR')
$outputHandler = [System.Diagnostics.DataReceivedEventHandler]$stdoutHandler.OnDataReceived
$errorHandler = [System.Diagnostics.DataReceivedEventHandler]$stderrHandler.OnDataReceived
$workerProcess.add_OutputDataReceived($outputHandler)
$workerProcess.add_ErrorDataReceived($errorHandler)
$workerProcess.BeginOutputReadLine()
$workerProcess.BeginErrorReadLine()
while (-not $workerProcess.HasExited) {
$commands = Get-PendingCommands
foreach ($command in $commands) {
$trimmed = $command.Trim()
if (-not $trimmed) { continue }
Write-ControllerLog "Received command '$trimmed'"
try {
$workerProcess.StandardInput.WriteLine($trimmed)
$workerProcess.StandardInput.Flush()
}
catch {
Write-ControllerLog "Failed to forward command '$trimmed': $($_.Exception.Message)"
}
if ($trimmed -ieq 'quit') {
Write-ControllerLog "Quit command issued. Waiting for worker to exit."
}
}
Start-Sleep -Milliseconds 500
}
$exitCode = $workerProcess.ExitCode
Write-ControllerLog "Worker exited with code $exitCode"
Write-Metadata -Status 'stopped' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount -LastExitCode $exitCode
if ($exitCode -eq 0) {
break
}
}
catch {
Write-ControllerLog "Controller error: $($_.Exception.Message)"
}
$restartCount++
if ($restartCount -gt $MaxRestarts) {
Write-ControllerLog "Maximum restart attempts reached. Controller stopping."
break
}
Write-ControllerLog "Restarting worker in $RestartDelaySeconds seconds (attempt $restartCount of $MaxRestarts)."
Start-Sleep -Seconds $RestartDelaySeconds
} }
Write-ControllerLog "Restarting worker in $RestartDelaySeconds seconds (attempt $restartCount of $MaxRestarts)." Write-Metadata -Status 'inactive' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount
Start-Sleep -Seconds $RestartDelaySeconds Write-ControllerLog "Controller exiting."
} }
catch {
Write-Metadata -Status 'inactive' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount Write-FatalLog "Fatal controller error: $($_.Exception.Message)"
Write-ControllerLog "Controller exiting." if ($_.ScriptStackTrace) {
Write-FatalLog "Stack: $($_.ScriptStackTrace)"
if ($logWriter) { }
$logWriter.Dispose() throw
} }
if ($logStream) { finally {
$logStream.Dispose() if ($logWriter) {
$logWriter.Dispose()
}
if ($logStream) {
$logStream.Dispose()
}
} }