begin persistence/attachment overhaul

This commit is contained in:
Nathan
2025-11-25 14:51:22 -07:00
parent 0b8f4f30fa
commit 2aa9061114
6 changed files with 2448 additions and 393 deletions

View File

@@ -50,6 +50,205 @@ $workers = @(
}
)
$global:UnifiedWorkerDataRoot = 'C:\ProgramData\UnifiedWorkers'
$script:ControllerScriptBase64 = $null
$script:AttachHelperScriptBase64 = $null
function Get-FileBase64 {
param([string]$Path)
[Convert]::ToBase64String([IO.File]::ReadAllBytes($Path))
}
function Get-ControllerScriptBase64 {
if (-not $script:ControllerScriptBase64) {
$controllerPath = Join-Path $PSScriptRoot 'remote_worker_controller.ps1'
$script:ControllerScriptBase64 = Get-FileBase64 -Path $controllerPath
}
return $script:ControllerScriptBase64
}
function Get-AttachHelperScriptBase64 {
if (-not $script:AttachHelperScriptBase64) {
$helperPath = Join-Path $PSScriptRoot 'remote_worker_attach.ps1'
$script:AttachHelperScriptBase64 = Get-FileBase64 -Path $helperPath
}
return $script:AttachHelperScriptBase64
}
function ConvertTo-Base64Unicode {
param([string]$Content)
[Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($Content))
}
function Get-WorkerSshArgs {
param([string]$RawArgs)
if ([string]::IsNullOrWhiteSpace($RawArgs)) {
return @()
}
return $RawArgs -split '\s+' | Where-Object { $_.Trim().Length -gt 0 }
}
function Invoke-RemotePowerShell {
param(
[Parameter(Mandatory = $true)][object]$Worker,
[Parameter(Mandatory = $true)][string]$Script
)
$bytes = [Text.Encoding]::Unicode.GetBytes($Script)
$encoded = [Convert]::ToBase64String($bytes)
$sshArgs = @('-o','ServerAliveInterval=60','-o','ServerAliveCountMax=30')
$sshArgs += (Get-WorkerSshArgs -RawArgs $Worker.SSHArgs)
$sshArgs += "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand $encoded"
& ssh @sshArgs
return $LASTEXITCODE
}
function Ensure-ControllerDeployed {
param([object]$Worker)
$controllerBase64 = Get-ControllerScriptBase64
$script = @'
`$dataRoot = '{0}'
New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$controllerPath = Join-Path `$dataRoot 'controller.ps1'
[IO.File]::WriteAllBytes(`$controllerPath, [Convert]::FromBase64String('{1}'))
Write-Host "Controller ready at `$controllerPath"
'@ -f $global:UnifiedWorkerDataRoot, $controllerBase64
Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
}
function Ensure-AttachHelperDeployed {
param([object]$Worker)
$helperBase64 = Get-AttachHelperScriptBase64
$script = @'
`$dataRoot = '{0}'
New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$attachPath = Join-Path `$dataRoot 'attach-helper.ps1'
[IO.File]::WriteAllBytes(`$attachPath, [Convert]::FromBase64String('{1}'))
'@ -f $global:UnifiedWorkerDataRoot, $helperBase64
Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
}
function Get-RemoteWorkerPaths {
param(
[string]$WorkerType,
[string]$WorkerName
)
$base = Join-Path $global:UnifiedWorkerDataRoot $WorkerType
$instance = Join-Path $base $WorkerName
return [pscustomobject]@{
InstanceRoot = $instance
MetadataPath = Join-Path $instance 'state\worker-info.json'
CommandPath = Join-Path $instance 'state\commands.txt'
LogPath = Join-Path $instance 'logs\worker.log'
}
}
function Get-EnsureWorkerScript {
param(
[string]$WorkerName,
[string]$WorkerType,
[string]$PayloadBase64
)
$paths = Get-RemoteWorkerPaths -WorkerType $WorkerType -WorkerName $WorkerName
$controllerPath = Join-Path $global:UnifiedWorkerDataRoot 'controller.ps1'
return @'
`$controllerPath = '{0}'
`$metaPath = '{1}'
`$workerName = '{2}'
`$workerType = '{3}'
`$payloadBase64 = '{4}'
if (-not (Test-Path `$controllerPath)) {
throw "Controller missing at `$controllerPath"
}
`$shouldStart = `$true
if (Test-Path `$metaPath) {
try {
`$meta = Get-Content `$metaPath -Raw | ConvertFrom-Json
if (`$meta.Status -eq 'running' -and `$meta.WorkerPid) {
if (Get-Process -Id `$meta.WorkerPid -ErrorAction SilentlyContinue) {
Write-Host "Worker `$workerName already running (PID `$($meta.WorkerPid))."
`$shouldStart = `$false
}
}
} catch {
Write-Host "Unable to parse metadata for `$workerName. Starting new controller." -ForegroundColor Yellow
}
}
if (`$shouldStart) {
`$psExe = (Get-Command pwsh -ErrorAction SilentlyContinue)?.Source
if (-not `$psExe) {
`$psExe = (Get-Command powershell -ErrorAction Stop).Source
}
`$controllerArgs = @(
'-NoLogo','-NoProfile','-ExecutionPolicy','Bypass',
'-File',"`$controllerPath",
'-WorkerName',"`$workerName",
'-WorkerType',"`$workerType",
'-PayloadBase64',"`$payloadBase64"
)
Start-Process -FilePath `$psExe -ArgumentList `$controllerArgs -WindowStyle Hidden | Out-Null
Write-Host "Worker `$workerName started under controller." -ForegroundColor Green
}
'@ -f $controllerPath, $paths.MetadataPath, $WorkerName, $WorkerType, $PayloadBase64
}
function Ensure-PersistentWorker {
param(
[object]$Worker,
[string]$WorkerType,
[string]$PayloadScript
)
Write-Host "[$($Worker.Name)] Ensuring worker is running under controller..." -ForegroundColor Cyan
Ensure-ControllerDeployed -Worker $Worker
$payloadBase64 = ConvertTo-Base64Unicode -Content $PayloadScript
$ensureScript = Get-EnsureWorkerScript -WorkerName $Worker.Name -WorkerType $WorkerType -PayloadBase64 $payloadBase64
$result = Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript
if ($result -ne 0) {
Write-Host "[$($Worker.Name)] Remote ensure command exited with code $result." -ForegroundColor Yellow
}
}
function Invoke-WorkerAttach {
param(
[object]$Worker,
[string]$WorkerType,
[switch]$CommandOnly,
[string]$Command
)
Ensure-AttachHelperDeployed -Worker $Worker
$paramsBlock = @("-WorkerName","$($Worker.Name)","-WorkerType","$WorkerType")
if ($CommandOnly) {
$paramsBlock += "-CommandOnly"
}
if ($Command) {
$paramsBlock += "-Command"
$paramsBlock += $Command
}
$paramsLiteral = ($paramsBlock | ForEach-Object { "'$_'" }) -join ', '
$script = @'
`$helperPath = Join-Path '{0}' 'attach-helper.ps1'
if (-not (Test-Path `$helperPath)) {
throw "Attach helper missing at `$helperPath"
}
`$arguments = @({1})
& `$helperPath @arguments
'@ -f $global:UnifiedWorkerDataRoot, $paramsLiteral
Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
}
# FUNCTIONS
# This function generates the standard PowerShell remote command
@@ -258,57 +457,10 @@ function Start-StandardWorker {
[Parameter(Mandatory = $true)]
[object]$Worker
)
$retryCount = 0
$retryDelay = 15 # seconds between retries
$workerRestarted = $false
while ($true) { # Changed to infinite loop
if ($retryCount -gt 0) {
Write-Host "`nRestarting worker process (Attempt $($retryCount + 1))..." -ForegroundColor Yellow
Start-Sleep -Seconds $retryDelay
$workerRestarted = $true
}
Write-Host "Connecting to $($Worker.Name)..." -ForegroundColor Cyan
if ($workerRestarted) {
Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow
}
try {
$remoteCommand = Get-RemoteStandardWorkerCommand
# Encode the command to handle special characters
$bytes = [System.Text.Encoding]::Unicode.GetBytes($remoteCommand)
$encodedCommand = [Convert]::ToBase64String($bytes)
# Execute the encoded command on the remote machine
Write-Host "Connecting to $($Worker.Name) and executing worker script..." -ForegroundColor Yellow
# Add SSH keepalive settings to reduce chance of random disconnections
$sshCommand = "ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($Worker.SSHArgs) ""powershell -EncodedCommand $encodedCommand"""
# Execute the SSH command and capture the exit code
Invoke-Expression $sshCommand
$sshExitCode = $LASTEXITCODE
# Check if SSH command completed successfully
if ($sshExitCode -eq 0) {
Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green
Start-Sleep -Seconds 2 # Brief pause before restarting
$retryCount = 0 # Reset counter for successful completion
continue # Continue the loop instead of breaking
} else {
throw "Worker process exited with code: $sshExitCode"
}
}
catch {
$retryCount++
Write-Host "`nAn error occurred while running worker on $($Worker.Name):" -ForegroundColor Red
Write-Host $_.Exception.Message -ForegroundColor Red
Write-Host "`nAttempting to restart worker in $retryDelay seconds..." -ForegroundColor Yellow
}
}
$payloadScript = Get-RemoteStandardWorkerCommand
Ensure-PersistentWorker -Worker $Worker -WorkerType 'flamenco' -PayloadScript $payloadScript
Invoke-WorkerAttach -Worker $Worker -WorkerType 'flamenco'
}
# This function launches the CMD worker
@@ -317,57 +469,10 @@ function Start-CmdWorker {
[Parameter(Mandatory = $true)]
[object]$Worker
)
$retryCount = 0
$retryDelay = 5 # seconds between retries
$workerRestarted = $false
while ($true) { # Changed to infinite loop
if ($retryCount -gt 0) {
Write-Host "`nRestarting worker process (Attempt $($retryCount + 1))..." -ForegroundColor Yellow
Start-Sleep -Seconds $retryDelay
$workerRestarted = $true
}
Write-Host "Connecting to $($Worker.Name) (CMD mode)..." -ForegroundColor Cyan
if ($workerRestarted) {
Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow
}
try {
$remoteCommand = Get-RemoteCmdWorkerCommand
# Encode the command to handle special characters
$bytes = [System.Text.Encoding]::Unicode.GetBytes($remoteCommand)
$encodedCommand = [Convert]::ToBase64String($bytes)
# Execute the encoded command on the remote machine
Write-Host "Connecting to $($Worker.Name) and executing CMD worker script..." -ForegroundColor Yellow
# Add SSH keepalive settings to reduce chance of random disconnections
$sshCommand = "ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($Worker.SSHArgs) ""powershell -EncodedCommand $encodedCommand"""
# Execute the SSH command and capture the exit code
Invoke-Expression $sshCommand
$sshExitCode = $LASTEXITCODE
# Check if SSH command completed successfully
if ($sshExitCode -eq 0) {
Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green
Start-Sleep -Seconds 2 # Brief pause before restarting
$retryCount = 0 # Reset counter for successful completion
continue # Continue the loop instead of breaking
} else {
throw "Worker process exited with code: $sshExitCode"
}
}
catch {
$retryCount++
Write-Host "`nAn error occurred while running worker on $($Worker.Name):" -ForegroundColor Red
Write-Host $_.Exception.Message -ForegroundColor Red
Write-Host "`nAttempting to restart worker in $retryDelay seconds..." -ForegroundColor Yellow
}
}
$payloadScript = Get-RemoteCmdWorkerCommand
Ensure-PersistentWorker -Worker $Worker -WorkerType 'flamenco' -PayloadScript $payloadScript
Invoke-WorkerAttach -Worker $Worker -WorkerType 'flamenco'
}
# This function launches ALL workers in Windows Terminal tabs
@@ -376,174 +481,22 @@ function Start-AllWorkers {
[Parameter(Mandatory = $true)]
[string]$WorkerType
)
Write-Host "Launching ALL $WorkerType workers in Windows Terminal tabs..." -ForegroundColor Cyan
try {
# First, check if Windows Terminal is available
if (-not (Get-Command wt.exe -ErrorAction SilentlyContinue)) {
Write-Host "Windows Terminal (wt.exe) not found. Falling back to separate windows." -ForegroundColor Yellow
$useTerminal = $false
} else {
$useTerminal = $true
}
foreach ($worker in $workers) {
# Create a new PowerShell script with a unique name for this worker
$tempScriptPath = [System.IO.Path]::GetTempFileName() + ".ps1"
# Create different script content based on worker type
if ($WorkerType -eq "CMD") {
# CMD workers get retry logic at the local level
$scriptContent = @"
# Wrap everything in a try-catch to prevent script termination
try {
Write-Host 'Launching $WorkerType worker for $($worker.Name)' -ForegroundColor Cyan
Write-Host "Ensuring ALL $WorkerType workers are running under controllers..." -ForegroundColor Cyan
`$retryCount = 0
`$retryDelay = 5 # seconds between retries
`$workerRestarted = `$false
while (`$true) { # Changed to infinite loop
try {
if (`$retryCount -gt 0) {
Write-Host "`nRestarting worker process (Attempt `$(`$retryCount + 1))..." -ForegroundColor Yellow
Start-Sleep -Seconds `$retryDelay
`$workerRestarted = `$true
}
Write-Host "Connecting to $($worker.Name) ($WorkerType mode)..." -ForegroundColor Cyan
if (`$workerRestarted) {
Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow
}
# Get remote command
`$remoteCommand = @'
$(Get-RemoteSimplifiedCmdWorkerCommand)
'@
# Encode the command
`$bytes = [System.Text.Encoding]::Unicode.GetBytes(`$remoteCommand)
`$encodedCommand = [Convert]::ToBase64String(`$bytes)
# Execute SSH command with keepalive settings and capture exit code
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($worker.SSHArgs) "powershell -EncodedCommand `$encodedCommand"
`$sshExitCode = `$LASTEXITCODE
# Check if SSH command completed successfully
if (`$sshExitCode -eq 0) {
Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green
Start-Sleep -Seconds 2 # Brief pause before restarting
`$retryCount = 0 # Reset counter for successful completion
continue # Continue the loop instead of breaking
} else {
throw "SSH command failed with exit code: `$sshExitCode"
}
}
catch {
`$retryCount++
Write-Host "An error occurred while connecting to $($worker.Name):" -ForegroundColor Red
Write-Host `$_.Exception.Message -ForegroundColor Red
Write-Host "Attempting to reconnect in `$retryDelay seconds..." -ForegroundColor Yellow
# Don't rethrow - we want to continue the retry loop
}
$payloadScript = if ($WorkerType -eq 'CMD') {
Get-RemoteCmdWorkerCommand
} else {
Get-RemoteStandardWorkerCommand
}
}
catch {
# This outer catch block is for any unexpected errors that might terminate the script
Write-Host "`nCRITICAL ERROR: Script encountered an unexpected error:" -ForegroundColor Red
Write-Host `$_.Exception.Message -ForegroundColor Red
Write-Host "`nRestarting the entire worker process in 5 seconds..." -ForegroundColor Yellow
Start-Sleep -Seconds 5
# Restart the script by calling itself
& `$MyInvocation.MyCommand.Path
}
"@
} else {
# Standard workers keep the original retry logic
$scriptContent = @"
# Wrap everything in a try-catch to prevent script termination
try {
Write-Host 'Launching $WorkerType worker for $($worker.Name)' -ForegroundColor Cyan
`$retryCount = 0
`$retryDelay = 5 # seconds between retries
while (`$true) { # Changed to infinite loop
try {
if (`$retryCount -gt 0) {
Write-Host "Retry attempt `$retryCount..." -ForegroundColor Yellow
Start-Sleep -Seconds `$retryDelay
}
# Get remote command
`$remoteCommand = @'
$(Get-RemoteStandardWorkerCommand)
'@
# Encode the command
`$bytes = [System.Text.Encoding]::Unicode.GetBytes(`$remoteCommand)
`$encodedCommand = [Convert]::ToBase64String(`$bytes)
# Execute SSH command with keepalive settings
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($worker.SSHArgs) "powershell -EncodedCommand `$encodedCommand"
`$sshExitCode = `$LASTEXITCODE
# Check SSH exit code and handle accordingly
if (`$sshExitCode -eq 0) {
Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green
Start-Sleep -Seconds 2 # Brief pause before restarting
`$retryCount = 0 # Reset counter for successful completion
continue # Continue the loop instead of breaking
} else {
throw "SSH command failed with exit code: `$sshExitCode"
}
}
catch {
`$retryCount++
Write-Host "An error occurred while connecting to $($worker.Name):" -ForegroundColor Red
Write-Host `$_.Exception.Message -ForegroundColor Red
Write-Host "Attempting to reconnect in `$retryDelay seconds..." -ForegroundColor Yellow
# Don't rethrow - we want to continue the retry loop
}
foreach ($worker in $workers) {
Ensure-PersistentWorker -Worker $worker -WorkerType 'flamenco' -PayloadScript $payloadScript
}
}
catch {
# This outer catch block is for any unexpected errors that might terminate the script
Write-Host "`nCRITICAL ERROR: Script encountered an unexpected error:" -ForegroundColor Red
Write-Host `$_.Exception.Message -ForegroundColor Red
Write-Host "`nRestarting the entire worker process in 5 seconds..." -ForegroundColor Yellow
Start-Sleep -Seconds 5
# Restart the script by calling itself
& `$MyInvocation.MyCommand.Path
}
"@
}
# Write the script to file
Set-Content -Path $tempScriptPath -Value $scriptContent
if ($useTerminal) {
# Launch in a new Windows Terminal tab
$tabTitle = "$($worker.Name) - $WorkerType Worker"
Start-Process wt.exe -ArgumentList "-w 0 new-tab --title `"$tabTitle`" powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$tempScriptPath`""
} else {
# Fallback to separate window if Windows Terminal is not available
Start-Process powershell -ArgumentList "-NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$tempScriptPath`""
}
Write-Host "Started $($worker.Name) ($WorkerType) worker in a new tab." -ForegroundColor Green
Start-Sleep -Milliseconds 300 # Small delay between launches
}
Write-Host "`nAll $WorkerType worker scripts have been launched in Windows Terminal tabs." -ForegroundColor Cyan
}
catch {
Write-Host "Error launching workers: $($_.Exception.Message)" -ForegroundColor Red
}
Write-Host "All workers processed. Attach to any worker from the menu to monitor output." -ForegroundColor Green
Write-Host "Press Enter to return to the menu..." -ForegroundColor Green
Read-Host
Read-Host | Out-Null
}
# Main menu loop