attachment is clean

This commit is contained in:
2025-12-01 15:52:35 -07:00
parent b61baac09b
commit 18683b4ad0
3 changed files with 2923 additions and 18 deletions

File diff suppressed because one or more lines are too long

View File

@@ -30,7 +30,7 @@ New-Item -ItemType Directory -Path $stateRoot -Force | Out-Null
$logPath = Join-Path -Path $logsRoot -ChildPath 'worker.log' $logPath = Join-Path -Path $logsRoot -ChildPath 'worker.log'
$metaPath = Join-Path -Path $stateRoot -ChildPath 'worker-info.json' $metaPath = Join-Path -Path $stateRoot -ChildPath 'worker-info.json'
$commandPath = Join-Path -Path $stateRoot -ChildPath 'commands.txt' $commandPath = Join-Path -Path $stateRoot -ChildPath 'commands.txt'
$payloadPath = Join-Path -Path $stateRoot -ChildPath "payload-$([Guid]::NewGuid().ToString()).ps1" $payloadPath = Join-Path -Path $stateRoot -ChildPath "payload.ps1"
# endregion # endregion
# region Helpers # region Helpers
@@ -82,6 +82,9 @@ function Get-PendingCommands {
} }
# endregion # endregion
# record initial state before launching worker
Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0
try { try {
# Write payload script to disk # Write payload script to disk
$payloadBytes = [Convert]::FromBase64String($PayloadBase64) $payloadBytes = [Convert]::FromBase64String($PayloadBase64)

View File

@@ -36,6 +36,53 @@ $script:ControllerScriptBase64 = $null
$script:AttachHelperScriptBase64 = $null $script:AttachHelperScriptBase64 = $null
$script:WorkerBasePathCache = @{} $script:WorkerBasePathCache = @{}
function Remove-ClixmlNoise {
param([object[]]$Lines)
$noisePatterns = @(
'^#<\s*CLIXML',
'^\s*<Objs\b', '^\s*</Objs>',
'^\s*<Obj\b', '^\s*</Obj>',
'^\s*<TN\b', '^\s*</TN>',
'^\s*<MS\b', '^\s*</MS>',
'^\s*<PR\b', '^\s*</PR>',
'^\s*<I64\b', '^\s*</I64>',
'^\s*<AI\b', '^\s*</AI>',
'^\s*<Nil\b', '^\s*</Nil>',
'^\s*<PI\b', '^\s*</PI>',
'^\s*<PC\b', '^\s*</PC>',
'^\s*<SR\b', '^\s*</SR>',
'^\s*<SD\b', '^\s*</SD>',
'^\s*<S\b', '^\s*</S>'
)
$filtered = @()
foreach ($entry in $Lines) {
if ($null -eq $entry) { continue }
$text = $entry.ToString()
$isNoise = $false
foreach ($pattern in $noisePatterns) {
if ($text -match $pattern) {
$isNoise = $true
break
}
}
if (-not $isNoise) {
$filtered += $text
}
}
return $filtered
}
function Write-FilteredSshOutput {
param([object[]]$Lines)
$clean = Remove-ClixmlNoise -Lines $Lines
foreach ($line in $clean) {
Write-Host $line
}
}
function Build-SshArgsFromParts { function Build-SshArgsFromParts {
param( param(
[pscustomobject]$Parts, [pscustomobject]$Parts,
@@ -102,8 +149,9 @@ function Get-WorkerBasePath {
$sshArgs = Build-SshArgsFromParts -Parts $ConnectionParts -Interactive:$false $sshArgs = Build-SshArgsFromParts -Parts $ConnectionParts -Interactive:$false
$scriptBlock = "`$ProgressPreference='SilentlyContinue'; [Environment]::GetFolderPath('LocalApplicationData')" $scriptBlock = "`$ProgressPreference='SilentlyContinue'; [Environment]::GetFolderPath('LocalApplicationData')"
$encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($scriptBlock)) $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($scriptBlock))
$remoteCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand $encoded" $remoteCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand $encoded"
$output = & ssh @sshArgs $remoteCmd $rawOutput = & ssh @sshArgs $remoteCmd 2>&1
$output = Remove-ClixmlNoise -Lines $rawOutput
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
throw "Unable to determine LocalAppData on $($Worker.Name)." throw "Unable to determine LocalAppData on $($Worker.Name)."
} }
@@ -230,11 +278,13 @@ function Invoke-RemotePowerShell {
$remoteTarget = "{0}:{1}" -f $parts.Host, ('"'+$remoteScriptScp+'"') $remoteTarget = "{0}:{1}" -f $parts.Host, ('"'+$remoteScriptScp+'"')
$ensureScript = "New-Item -ItemType Directory -Path '$remoteTmpDir' -Force | Out-Null" $ensureScript = "New-Item -ItemType Directory -Path '$remoteTmpDir' -Force | Out-Null"
$ensureCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($ensureScript)) $ensureCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($ensureScript))
& ssh @sshBaseArgs $ensureCmd $ensureOutput = & ssh @sshBaseArgs $ensureCmd 2>&1
if ($LASTEXITCODE -ne 0) { $ensureExit = $LASTEXITCODE
Write-FilteredSshOutput -Lines $ensureOutput
if ($ensureExit -ne 0) {
Remove-Item $localTemp -ErrorAction SilentlyContinue Remove-Item $localTemp -ErrorAction SilentlyContinue
return $LASTEXITCODE return $ensureExit
} }
$scpArgs = Build-ScpArgsFromParts -Parts $parts $scpArgs = Build-ScpArgsFromParts -Parts $parts
@@ -248,15 +298,47 @@ function Invoke-RemotePowerShell {
return $scpExit return $scpExit
} }
$execCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$remoteScriptWin`"" $execCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -File `"$remoteScriptWin`""
& ssh @sshBaseArgs $execCmd $execOutput = & ssh @sshBaseArgs $execCmd 2>&1
$execExit = $LASTEXITCODE $execExit = $LASTEXITCODE
Write-FilteredSshOutput -Lines $execOutput
$cleanupScript = "Remove-Item -LiteralPath '$remoteScriptWin' -ErrorAction SilentlyContinue" $cleanupScript = "Remove-Item -LiteralPath '$remoteScriptWin' -ErrorAction SilentlyContinue"
$cleanupCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($cleanupScript)) $cleanupCmd = "powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($cleanupScript))
& ssh @sshBaseArgs $cleanupCmd | Out-Null $cleanupOutput = & ssh @sshBaseArgs $cleanupCmd 2>&1
Write-FilteredSshOutput -Lines $cleanupOutput
return $execExit return [int]$execExit
}
function Resolve-ExitCode {
param($Value)
if ($Value -is [System.Array]) {
for ($i = $Value.Count - 1; $i -ge 0; $i--) {
$candidate = Resolve-ExitCode -Value $Value[$i]
if ($candidate -ne $null) {
return $candidate
}
}
return 0
}
if ($Value -is [int]) {
return $Value
}
$text = $Value
if ($null -eq $text) {
return 0
}
$parsed = 0
if ([int]::TryParse($text.ToString(), [ref]$parsed)) {
return $parsed
}
return 0
} }
function Ensure-ControllerDeployed { function Ensure-ControllerDeployed {
@@ -269,7 +351,10 @@ New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$controllerPath = Join-Path `$dataRoot 'controller.ps1' `$controllerPath = Join-Path `$dataRoot 'controller.ps1'
[IO.File]::WriteAllBytes(`$controllerPath, [Convert]::FromBase64String('$controllerBase64')) [IO.File]::WriteAllBytes(`$controllerPath, [Convert]::FromBase64String('$controllerBase64'))
"@ "@
Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $script)
if ($exit -ne 0) {
throw "Controller deployment failed on $($Worker.Name) (exit $exit)."
}
} }
function Ensure-AttachHelperDeployed { function Ensure-AttachHelperDeployed {
@@ -282,7 +367,10 @@ New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$attachPath = Join-Path `$dataRoot 'attach-helper.ps1' `$attachPath = Join-Path `$dataRoot 'attach-helper.ps1'
[IO.File]::WriteAllBytes(`$attachPath, [Convert]::FromBase64String('$helperBase64')) [IO.File]::WriteAllBytes(`$attachPath, [Convert]::FromBase64String('$helperBase64'))
"@ "@
Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $script)
if ($exit -ne 0) {
throw "Attach helper deployment failed on $($Worker.Name) (exit $exit)."
}
} }
function Get-EnsureWorkerScript { function Get-EnsureWorkerScript {
@@ -308,6 +396,14 @@ function Get-EnsureWorkerScript {
`$payloadBase64 = `$params.PayloadBase64 `$payloadBase64 = `$params.PayloadBase64
`$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers' `$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
`$instanceRoot = Join-Path (Join-Path `$dataRoot `$workerType) `$workerName `$instanceRoot = Join-Path (Join-Path `$dataRoot `$workerType) `$workerName
`$logsRoot = Join-Path `$instanceRoot 'logs'
`$stateRoot = Join-Path `$instanceRoot 'state'
New-Item -ItemType Directory -Path `$logsRoot -Force | Out-Null
New-Item -ItemType Directory -Path `$stateRoot -Force | Out-Null
`$logPath = Join-Path `$logsRoot 'worker.log'
`$commandPath = Join-Path `$stateRoot 'commands.txt'
if (-not (Test-Path `$logPath)) { New-Item -Path `$logPath -ItemType File -Force | Out-Null }
if (-not (Test-Path `$commandPath)) { New-Item -Path `$commandPath -ItemType File -Force | Out-Null }
`$metaPath = Join-Path `$instanceRoot 'state\worker-info.json' `$metaPath = Join-Path `$instanceRoot 'state\worker-info.json'
`$controllerPath = Join-Path `$dataRoot 'controller.ps1' `$controllerPath = Join-Path `$dataRoot 'controller.ps1'
@@ -331,6 +427,21 @@ if (Test-Path `$metaPath) {
} }
if (`$shouldStart) { if (`$shouldStart) {
`$initialMeta = [pscustomobject]@{
WorkerName = `$workerName
WorkerType = `$workerType
Status = 'launching'
ControllerPid = `$null
WorkerPid = `$null
Restarts = 0
LastExitCode = `$null
LogPath = `$logPath
CommandPath = `$commandPath
PayloadPath = `$null
UpdatedAtUtc = (Get-Date).ToUniversalTime()
} | ConvertTo-Json -Depth 5
`$initialMeta | Set-Content -Path `$metaPath -Encoding UTF8
`$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue `$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
if (`$pwsh) { if (`$pwsh) {
`$psExe = `$pwsh.Source `$psExe = `$pwsh.Source
@@ -363,7 +474,10 @@ function Ensure-PersistentWorker {
Ensure-ControllerDeployed -Worker $Worker Ensure-ControllerDeployed -Worker $Worker
$payloadBase64 = ConvertTo-Base64Unicode -Content $PayloadScript $payloadBase64 = ConvertTo-Base64Unicode -Content $PayloadScript
$ensureScript = Get-EnsureWorkerScript -WorkerName $Worker.Name -WorkerType $WorkerType -PayloadBase64 $payloadBase64 $ensureScript = Get-EnsureWorkerScript -WorkerName $Worker.Name -WorkerType $WorkerType -PayloadBase64 $payloadBase64
Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript | Out-Null $exit = Resolve-ExitCode (Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript)
if ($exit -ne 0) {
throw "Worker ensure script failed on $($Worker.Name) (exit $exit)."
}
} }
function Test-WorkerMetadataExists { function Test-WorkerMetadataExists {
@@ -561,11 +675,21 @@ function Ensure-SheepItWorkerController {
function Start-SheepItWorker { function Start-SheepItWorker {
param([object]$Worker) param([object]$Worker)
try {
Write-Host "Ensuring SheepIt controller on $($Worker.Name)..." -ForegroundColor Cyan
Ensure-SheepItWorkerController -Worker $Worker Ensure-SheepItWorkerController -Worker $Worker
if (-not (Wait-WorkerMetadata -Worker $Worker -WorkerType 'sheepit' -TimeoutSeconds 20)) { }
Write-Host "Worker metadata did not appear on $($Worker.Name). Check controller logs." -ForegroundColor Red catch {
Write-Host "Failed to ensure controller on $($Worker.Name): $($_.Exception.Message)" -ForegroundColor Red
return return
} }
if (-not (Wait-WorkerMetadata -Worker $Worker -WorkerType 'sheepit' -TimeoutSeconds 30)) {
Write-Host "Worker metadata did not appear on $($Worker.Name). Check controller logs under %LocalAppData%\UnifiedWorkers." -ForegroundColor Red
return
}
Write-Host "Controller ready. Attaching to SheepIt worker on $($Worker.Name)..." -ForegroundColor Cyan
Invoke-WorkerAttach -Worker $Worker -WorkerType 'sheepit' Invoke-WorkerAttach -Worker $Worker -WorkerType 'sheepit'
} }