begin application build overhaul
This commit is contained in:
Nathan
116
Scripts/remote_worker_attach.ps1
Normal file
116
Scripts/remote_worker_attach.ps1
Normal file
@@ -0,0 +1,116 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkerName,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkerType,
|
||||
|
||||
[string]$DataRoot = (Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'),
|
||||
|
||||
[switch]$CommandOnly,
|
||||
|
||||
[string]$Command
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Get-WorkerPaths {
|
||||
param([string]$Root, [string]$Type, [string]$Name)
|
||||
|
||||
$instanceRoot = Join-Path -Path (Join-Path -Path $Root -ChildPath $Type) -ChildPath $Name
|
||||
return [pscustomobject]@{
|
||||
Metadata = Join-Path -Path $instanceRoot -ChildPath 'state\worker-info.json'
|
||||
Command = Join-Path -Path $instanceRoot -ChildPath 'state\commands.txt'
|
||||
Log = Join-Path -Path $instanceRoot -ChildPath 'logs\worker.log'
|
||||
}
|
||||
}
|
||||
|
||||
$paths = Get-WorkerPaths -Root $DataRoot -Type $WorkerType -Name $WorkerName
|
||||
|
||||
if (-not (Test-Path $paths.Metadata)) {
|
||||
Write-Host "No worker metadata found for $WorkerName ($WorkerType)." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
try {
|
||||
$metadata = Get-Content -Path $paths.Metadata -Raw | ConvertFrom-Json
|
||||
}
|
||||
catch {
|
||||
Write-Host "Unable to read worker metadata: $($_.Exception.Message)" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
if (Test-Path $paths.Log) {
|
||||
# ensure log file exists but do not truncate
|
||||
$null = (Get-Item $paths.Log)
|
||||
} else {
|
||||
New-Item -Path $paths.Log -ItemType File -Force | Out-Null
|
||||
}
|
||||
|
||||
if (-not (Test-Path $paths.Command)) {
|
||||
New-Item -Path $paths.Command -ItemType File -Force | Out-Null
|
||||
}
|
||||
|
||||
function Send-WorkerCommand {
|
||||
param([string]$Value)
|
||||
|
||||
$clean = $Value.Trim()
|
||||
if (-not $clean) {
|
||||
return
|
||||
}
|
||||
|
||||
Add-Content -Path $paths.Command -Value $clean -Encoding UTF8
|
||||
Write-Host "Sent command '$clean' to $WorkerName." -ForegroundColor DarkGray
|
||||
}
|
||||
|
||||
if ($CommandOnly) {
|
||||
if (-not $Command) {
|
||||
Write-Host "CommandOnly flag set but no command provided." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Send-WorkerCommand -Value $Command
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "Attaching to $WorkerName ($WorkerType) logs." -ForegroundColor Cyan
|
||||
Write-Host "Type commands and press Enter. Type 'detach' to exit session." -ForegroundColor Yellow
|
||||
|
||||
$logJob = Start-Job -ScriptBlock {
|
||||
param($LogPath)
|
||||
Get-Content -Path $LogPath -Tail 50 -Wait
|
||||
} -ArgumentList $paths.Log
|
||||
|
||||
try {
|
||||
while ($true) {
|
||||
$input = Read-Host "> "
|
||||
if ($null -eq $input) {
|
||||
continue
|
||||
}
|
||||
|
||||
$normalized = $input.Trim()
|
||||
if ($normalized.StartsWith(':')) {
|
||||
$normalized = $normalized.TrimStart(':').Trim()
|
||||
}
|
||||
|
||||
if ($normalized.Length -eq 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
if ($normalized.ToLowerInvariant() -eq 'detach') {
|
||||
break
|
||||
}
|
||||
|
||||
Send-WorkerCommand -Value $input
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($logJob) {
|
||||
Stop-Job -Job $logJob -ErrorAction SilentlyContinue
|
||||
Receive-Job -Job $logJob -ErrorAction SilentlyContinue | Out-Null
|
||||
Remove-Job -Job $logJob -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
Write-Host "Detached from worker $WorkerName." -ForegroundColor Cyan
|
||||
}
|
||||
|
||||
407
Scripts/remote_worker_controller.ps1
Normal file
407
Scripts/remote_worker_controller.ps1
Normal file
@@ -0,0 +1,407 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkerName,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkerType,
|
||||
|
||||
[string]$PayloadBase64,
|
||||
|
||||
[string]$PayloadBase64Path,
|
||||
|
||||
[string]$DataRoot = (Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'),
|
||||
|
||||
[int]$MaxRestarts = 5,
|
||||
|
||||
[int]$RestartDelaySeconds = 10
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
try {
|
||||
if ($Host -and $Host.Runspace) {
|
||||
[System.Management.Automation.Runspaces.Runspace]::DefaultRunspace = $Host.Runspace
|
||||
}
|
||||
}
|
||||
catch {
|
||||
# Ignore runspace assignment errors - not critical for non-interactive execution
|
||||
}
|
||||
|
||||
# region Path setup
|
||||
$workerRoot = Join-Path -Path $DataRoot -ChildPath $WorkerType
|
||||
$instanceRoot = Join-Path -Path $workerRoot -ChildPath $WorkerName
|
||||
New-Item -ItemType Directory -Path $instanceRoot -Force | Out-Null
|
||||
|
||||
$logsRoot = Join-Path -Path $instanceRoot -ChildPath 'logs'
|
||||
$stateRoot = Join-Path -Path $instanceRoot -ChildPath 'state'
|
||||
New-Item -ItemType Directory -Path $logsRoot -Force | Out-Null
|
||||
New-Item -ItemType Directory -Path $stateRoot -Force | Out-Null
|
||||
|
||||
$logPath = Join-Path -Path $logsRoot -ChildPath 'worker.log'
|
||||
$metaPath = Join-Path -Path $stateRoot -ChildPath 'worker-info.json'
|
||||
$commandPath = Join-Path -Path $stateRoot -ChildPath 'commands.txt'
|
||||
$payloadPath = Join-Path -Path $stateRoot -ChildPath "payload.ps1"
|
||||
# endregion
|
||||
|
||||
# region Logging
|
||||
try {
|
||||
$logStream = [System.IO.FileStream]::new(
|
||||
$logPath,
|
||||
[System.IO.FileMode]::Append,
|
||||
[System.IO.FileAccess]::Write,
|
||||
[System.IO.FileShare]::ReadWrite
|
||||
)
|
||||
$logWriter = [System.IO.StreamWriter]::new($logStream, [System.Text.Encoding]::UTF8)
|
||||
$logWriter.AutoFlush = $true
|
||||
}
|
||||
catch {
|
||||
# If we can't open the log file, write error to metadata and exit
|
||||
$errorMeta = [pscustomobject]@{
|
||||
WorkerName = $WorkerName
|
||||
WorkerType = $WorkerType
|
||||
Status = 'error'
|
||||
ControllerPid = $PID
|
||||
WorkerPid = $null
|
||||
Restarts = 0
|
||||
LastExitCode = 1
|
||||
LogPath = $logPath
|
||||
CommandPath = $commandPath
|
||||
PayloadPath = $payloadPath
|
||||
UpdatedAtUtc = (Get-Date).ToUniversalTime()
|
||||
ErrorMessage = "Failed to open log file: $($_.Exception.Message)"
|
||||
}
|
||||
$errorMeta | ConvertTo-Json -Depth 5 | Set-Content -Path $metaPath -Encoding UTF8 -ErrorAction SilentlyContinue
|
||||
Write-Error "Controller failed to initialize: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Create C# event handler class that doesn't require PowerShell runspace
|
||||
if (-not ("UnifiedWorkers.ProcessLogHandler" -as [type])) {
|
||||
$csharpCode = @'
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
||||
namespace UnifiedWorkers
|
||||
{
|
||||
public sealed class ProcessLogHandler
|
||||
{
|
||||
private readonly TextWriter _writer;
|
||||
private readonly string _prefix;
|
||||
private readonly object _lock = new object();
|
||||
|
||||
public ProcessLogHandler(TextWriter writer, string prefix)
|
||||
{
|
||||
_writer = writer ?? throw new ArgumentNullException("writer");
|
||||
_prefix = prefix ?? throw new ArgumentNullException("prefix");
|
||||
}
|
||||
|
||||
public void OnDataReceived(object sender, DataReceivedEventArgs e)
|
||||
{
|
||||
if (e == null || string.IsNullOrEmpty(e.Data))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
try
|
||||
{
|
||||
var timestamp = DateTime.UtcNow.ToString("u");
|
||||
_writer.WriteLine(string.Format("[{0} {1}] {2}", _prefix, timestamp, e.Data));
|
||||
_writer.Flush();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore write errors to prevent cascading failures
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'@
|
||||
Add-Type -TypeDefinition $csharpCode -ErrorAction Stop
|
||||
}
|
||||
|
||||
function Write-LogLine {
|
||||
param(
|
||||
[string]$Prefix,
|
||||
[string]$Message
|
||||
)
|
||||
|
||||
if (-not $logWriter) { return }
|
||||
$timestamp = (Get-Date).ToString('u')
|
||||
$logWriter.WriteLine("[$Prefix $timestamp] $Message")
|
||||
}
|
||||
|
||||
function Write-ControllerLog {
|
||||
param([string]$Message)
|
||||
Write-LogLine -Prefix 'CTRL' -Message $Message
|
||||
}
|
||||
|
||||
function Write-FatalLog {
|
||||
param([string]$Message)
|
||||
|
||||
try {
|
||||
Write-ControllerLog $Message
|
||||
}
|
||||
catch {
|
||||
$timestamp = (Get-Date).ToString('u')
|
||||
$fallback = "[CTRL $timestamp] $Message"
|
||||
try {
|
||||
[System.IO.File]::AppendAllText($logPath, $fallback + [Environment]::NewLine, [System.Text.Encoding]::UTF8)
|
||||
}
|
||||
catch {
|
||||
# last resort: write to host
|
||||
Write-Error $fallback
|
||||
}
|
||||
}
|
||||
}
|
||||
# endregion
|
||||
|
||||
# region Helpers
|
||||
|
||||
function Resolve-PayloadBase64 {
|
||||
if ($PayloadBase64) {
|
||||
return $PayloadBase64.Trim()
|
||||
}
|
||||
|
||||
if ($PayloadBase64Path) {
|
||||
if (-not (Test-Path $PayloadBase64Path)) {
|
||||
throw "Payload file '$PayloadBase64Path' not found."
|
||||
}
|
||||
|
||||
$content = Get-Content -Path $PayloadBase64Path -Raw
|
||||
if ([string]::IsNullOrWhiteSpace($content)) {
|
||||
throw "Payload file '$PayloadBase64Path' is empty."
|
||||
}
|
||||
|
||||
return $content.Trim()
|
||||
}
|
||||
|
||||
throw "No payload data provided to controller."
|
||||
}
|
||||
|
||||
function Write-Metadata {
|
||||
param(
|
||||
[string]$Status,
|
||||
[nullable[int]]$WorkerPid = $null,
|
||||
[nullable[int]]$ControllerPid = $PID,
|
||||
[int]$Restarts = 0,
|
||||
[nullable[int]]$LastExitCode = $null
|
||||
)
|
||||
|
||||
$payload = [pscustomobject]@{
|
||||
WorkerName = $WorkerName
|
||||
WorkerType = $WorkerType
|
||||
Status = $Status
|
||||
ControllerPid = $ControllerPid
|
||||
WorkerPid = $WorkerPid
|
||||
Restarts = $Restarts
|
||||
LastExitCode = $LastExitCode
|
||||
LogPath = $logPath
|
||||
CommandPath = $commandPath
|
||||
PayloadPath = $payloadPath
|
||||
UpdatedAtUtc = (Get-Date).ToUniversalTime()
|
||||
}
|
||||
|
||||
$payload | ConvertTo-Json -Depth 5 | Set-Content -Path $metaPath -Encoding UTF8
|
||||
}
|
||||
|
||||
function Get-PendingCommands {
|
||||
if (-not (Test-Path $commandPath)) {
|
||||
return @()
|
||||
}
|
||||
|
||||
try {
|
||||
$lines = Get-Content -Path $commandPath -ErrorAction Stop
|
||||
Remove-Item -Path $commandPath -Force -ErrorAction SilentlyContinue
|
||||
return $lines | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
|
||||
}
|
||||
catch {
|
||||
return @()
|
||||
}
|
||||
}
|
||||
# endregion
|
||||
|
||||
try {
|
||||
# record initial state before launching worker
|
||||
Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0
|
||||
|
||||
$resolvedPayloadBase64 = Resolve-PayloadBase64
|
||||
$PayloadBase64 = $resolvedPayloadBase64
|
||||
|
||||
try {
|
||||
# Write payload script to disk
|
||||
# The payload is base64-encoded UTF-16 (Unicode), so decode it properly
|
||||
Write-ControllerLog "Decoding payload base64 (length: $($resolvedPayloadBase64.Length))"
|
||||
$payloadBytes = [Convert]::FromBase64String($resolvedPayloadBase64)
|
||||
Write-ControllerLog "Decoded payload to $($payloadBytes.Length) bytes"
|
||||
|
||||
# Convert UTF-16 bytes back to string, then write as UTF-8 (PowerShell's preferred encoding)
|
||||
$payloadText = [Text.Encoding]::Unicode.GetString($payloadBytes)
|
||||
[IO.File]::WriteAllText($payloadPath, $payloadText, [Text.Encoding]::UTF8)
|
||||
Write-ControllerLog "Payload written to $payloadPath ($($payloadText.Length) characters)"
|
||||
}
|
||||
catch {
|
||||
Write-FatalLog "Unable to write payload: $($_.Exception.Message)"
|
||||
if ($_.Exception.InnerException) {
|
||||
Write-FatalLog "Inner exception: $($_.Exception.InnerException.Message)"
|
||||
}
|
||||
throw
|
||||
}
|
||||
|
||||
$restartCount = 0
|
||||
$controllerPid = $PID
|
||||
|
||||
while ($restartCount -le $MaxRestarts) {
|
||||
try {
|
||||
# Initialize worker process
|
||||
$psi = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
|
||||
if ($pwsh) {
|
||||
$psi.FileName = $pwsh.Source
|
||||
}
|
||||
else {
|
||||
$psi.FileName = (Get-Command powershell -ErrorAction Stop).Source
|
||||
}
|
||||
|
||||
$psi.Arguments = "-NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$payloadPath`""
|
||||
$psi.UseShellExecute = $false
|
||||
$psi.RedirectStandardInput = $true
|
||||
$psi.RedirectStandardOutput = $true
|
||||
$psi.RedirectStandardError = $true
|
||||
$psi.CreateNoWindow = $true
|
||||
|
||||
$workerProcess = New-Object System.Diagnostics.Process
|
||||
$workerProcess.StartInfo = $psi
|
||||
|
||||
if (-not $workerProcess.Start()) {
|
||||
throw "Failed to start worker process."
|
||||
}
|
||||
|
||||
Write-ControllerLog "Worker process started with PID $($workerProcess.Id)"
|
||||
Write-Metadata -Status 'running' -WorkerPid $workerProcess.Id -ControllerPid $controllerPid -Restarts $restartCount
|
||||
|
||||
# Check if process exited immediately
|
||||
if ($workerProcess.HasExited) {
|
||||
$exitCode = -1
|
||||
try {
|
||||
$exitCode = $workerProcess.ExitCode
|
||||
}
|
||||
catch {
|
||||
Write-ControllerLog "Unable to read immediate exit code: $($_.Exception.Message)"
|
||||
}
|
||||
Write-ControllerLog "Worker process exited immediately after start with code $exitCode"
|
||||
Write-Metadata -Status 'stopped' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount -LastExitCode $exitCode
|
||||
if ($exitCode -eq 0) { break }
|
||||
# Continue to restart logic below
|
||||
}
|
||||
else {
|
||||
$stdoutHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'OUT')
|
||||
$stderrHandler = [UnifiedWorkers.ProcessLogHandler]::new($logWriter, 'ERR')
|
||||
|
||||
$outputHandler = [System.Diagnostics.DataReceivedEventHandler]$stdoutHandler.OnDataReceived
|
||||
$errorHandler = [System.Diagnostics.DataReceivedEventHandler]$stderrHandler.OnDataReceived
|
||||
|
||||
$workerProcess.add_OutputDataReceived($outputHandler)
|
||||
$workerProcess.add_ErrorDataReceived($errorHandler)
|
||||
$workerProcess.BeginOutputReadLine()
|
||||
$workerProcess.BeginErrorReadLine()
|
||||
Write-ControllerLog "Output handlers set up successfully"
|
||||
|
||||
# Give process a moment to start, then check again
|
||||
Start-Sleep -Milliseconds 200
|
||||
if ($workerProcess.HasExited) {
|
||||
$exitCode = -1
|
||||
try {
|
||||
$exitCode = $workerProcess.ExitCode
|
||||
}
|
||||
catch {
|
||||
Write-ControllerLog "Unable to read exit code after delay: $($_.Exception.Message)"
|
||||
}
|
||||
Write-ControllerLog "Worker process exited after 200ms delay with code $exitCode"
|
||||
Write-Metadata -Status 'stopped' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount -LastExitCode $exitCode
|
||||
if ($exitCode -eq 0) { break }
|
||||
# Continue to restart logic below
|
||||
}
|
||||
else {
|
||||
Write-ControllerLog "Worker process is running, entering monitoring loop"
|
||||
|
||||
while (-not $workerProcess.HasExited) {
|
||||
$commands = Get-PendingCommands
|
||||
foreach ($command in $commands) {
|
||||
$trimmed = $command.Trim()
|
||||
if (-not $trimmed) { continue }
|
||||
|
||||
Write-ControllerLog "Received command '$trimmed'"
|
||||
try {
|
||||
$workerProcess.StandardInput.WriteLine($trimmed)
|
||||
$workerProcess.StandardInput.Flush()
|
||||
}
|
||||
catch {
|
||||
Write-ControllerLog "Failed to forward command '$trimmed': $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if ($trimmed -ieq 'quit') {
|
||||
Write-ControllerLog "Quit command issued. Waiting for worker to exit."
|
||||
}
|
||||
}
|
||||
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
# End of monitoring loop - process has exited
|
||||
Write-ControllerLog "Worker process exited, exiting monitoring loop"
|
||||
}
|
||||
}
|
||||
|
||||
# Wait for process to fully exit before reading exit code
|
||||
$workerProcess.WaitForExit(1000)
|
||||
|
||||
$exitCode = -1
|
||||
try {
|
||||
$exitCode = $workerProcess.ExitCode
|
||||
}
|
||||
catch {
|
||||
Write-ControllerLog "Unable to read worker exit code: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
Write-ControllerLog "Worker exited with code $exitCode"
|
||||
Write-Metadata -Status 'stopped' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount -LastExitCode $exitCode
|
||||
|
||||
if ($exitCode -eq 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-ControllerLog "Controller error: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
$restartCount++
|
||||
if ($restartCount -gt $MaxRestarts) {
|
||||
Write-ControllerLog "Maximum restart attempts reached. Controller stopping."
|
||||
break
|
||||
}
|
||||
|
||||
Write-ControllerLog "Restarting worker in $RestartDelaySeconds seconds (attempt $restartCount of $MaxRestarts)."
|
||||
Start-Sleep -Seconds $RestartDelaySeconds
|
||||
}
|
||||
|
||||
Write-Metadata -Status 'inactive' -WorkerPid $null -ControllerPid $controllerPid -Restarts $restartCount
|
||||
Write-ControllerLog "Controller exiting."
|
||||
}
|
||||
catch {
|
||||
Write-FatalLog "Fatal controller error: $($_.Exception.Message)"
|
||||
if ($_.ScriptStackTrace) {
|
||||
Write-FatalLog "Stack: $($_.ScriptStackTrace)"
|
||||
}
|
||||
throw
|
||||
}
|
||||
finally {
|
||||
if ($logWriter) {
|
||||
$logWriter.Dispose()
|
||||
}
|
||||
if ($logStream) {
|
||||
$logStream.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user