699 lines
29 KiB
C#
699 lines
29 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Reflection;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using UnifiedFarmLauncher.Models;
|
|
|
|
namespace UnifiedFarmLauncher.Services
|
|
{
|
|
public class WorkerControllerService
|
|
{
|
|
private readonly SshService _sshService;
|
|
private readonly ConfigService _configService;
|
|
private byte[]? _controllerScriptBytes;
|
|
private byte[]? _attachHelperScriptBytes;
|
|
|
|
public WorkerControllerService(SshService sshService, ConfigService configService)
|
|
{
|
|
_sshService = sshService;
|
|
_configService = configService;
|
|
}
|
|
|
|
private byte[] GetControllerScriptBytes()
|
|
{
|
|
if (_controllerScriptBytes != null)
|
|
return _controllerScriptBytes;
|
|
|
|
var assembly = Assembly.GetExecutingAssembly();
|
|
var resourceName = "UnifiedFarmLauncher.Scripts.remote_worker_controller.ps1";
|
|
|
|
using var stream = assembly.GetManifestResourceStream(resourceName);
|
|
if (stream == null)
|
|
throw new InvalidOperationException($"Resource {resourceName} not found");
|
|
|
|
using var reader = new BinaryReader(stream);
|
|
_controllerScriptBytes = reader.ReadBytes((int)stream.Length);
|
|
return _controllerScriptBytes;
|
|
}
|
|
|
|
private byte[] GetAttachHelperScriptBytes()
|
|
{
|
|
if (_attachHelperScriptBytes != null)
|
|
return _attachHelperScriptBytes;
|
|
|
|
var assembly = Assembly.GetExecutingAssembly();
|
|
var resourceName = "UnifiedFarmLauncher.Scripts.remote_worker_attach.ps1";
|
|
|
|
using var stream = assembly.GetManifestResourceStream(resourceName);
|
|
if (stream == null)
|
|
throw new InvalidOperationException($"Resource {resourceName} not found");
|
|
|
|
using var reader = new BinaryReader(stream);
|
|
_attachHelperScriptBytes = reader.ReadBytes((int)stream.Length);
|
|
return _attachHelperScriptBytes;
|
|
}
|
|
|
|
public async Task DeployControllerAsync(WorkerConfig worker)
|
|
{
|
|
var controllerBase64 = Convert.ToBase64String(GetControllerScriptBytes());
|
|
var script = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
New-Item -ItemType Directory -Path $dataRoot -Force | Out-Null
|
|
$controllerPath = Join-Path $dataRoot 'controller.ps1'
|
|
[IO.File]::WriteAllBytes($controllerPath, [Convert]::FromBase64String('{controllerBase64}'))
|
|
";
|
|
|
|
await _sshService.ExecuteRemoteScriptAsync(worker, script);
|
|
}
|
|
|
|
public async Task DeployAttachHelperAsync(WorkerConfig worker)
|
|
{
|
|
var helperBase64 = Convert.ToBase64String(GetAttachHelperScriptBytes());
|
|
var script = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
New-Item -ItemType Directory -Path $dataRoot -Force | Out-Null
|
|
$attachPath = Join-Path $dataRoot 'attach-helper.ps1'
|
|
[IO.File]::WriteAllBytes($attachPath, [Convert]::FromBase64String('{helperBase64}'))
|
|
";
|
|
|
|
await _sshService.ExecuteRemoteScriptAsync(worker, script);
|
|
}
|
|
|
|
public string GenerateSheepItPayload(WorkerConfig worker)
|
|
{
|
|
if (worker.WorkerTypes.SheepIt == null)
|
|
throw new InvalidOperationException("Worker does not have SheepIt configuration");
|
|
|
|
var config = _configService.Load();
|
|
var sheepIt = worker.WorkerTypes.SheepIt;
|
|
var globalSettings = config.GlobalSettings;
|
|
var safeKey = globalSettings.SheepItRenderKey.Replace("'", "''");
|
|
var safeUser = globalSettings.SheepItUsername.Replace("'", "''");
|
|
var urls = globalSettings.SheepItJarUrls;
|
|
var urlLiteral = "@(" + string.Join(", ", Array.ConvertAll(urls.ToArray(), url => $"'{url}'")) + ")";
|
|
|
|
return $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
# Write startup message to stderr so controller can capture it
|
|
[Console]::Error.WriteLine('[SHEEPIT] Payload script starting...')
|
|
|
|
try {{
|
|
$appData = [Environment]::GetFolderPath('ApplicationData')
|
|
$sheepDir = Join-Path $appData 'sheepit'
|
|
if (-not (Test-Path $sheepDir)) {{
|
|
New-Item -Path $sheepDir -ItemType Directory -Force | Out-Null
|
|
}}
|
|
|
|
$jarPath = Join-Path $sheepDir 'sheepit-client.jar'
|
|
$urls = {urlLiteral}
|
|
$headers = @{{ 'User-Agent' = 'Mozilla/5.0' }}
|
|
|
|
if (Test-Path $jarPath) {{
|
|
Write-Host ""SheepIt client already present at $jarPath. Skipping download."" -ForegroundColor Green
|
|
}}
|
|
else {{
|
|
$downloaded = $false
|
|
|
|
foreach ($url in $urls) {{
|
|
Write-Host ""Downloading SheepIt client from $url..."" -ForegroundColor Cyan
|
|
try {{
|
|
Invoke-WebRequest -Uri $url -OutFile $jarPath -UseBasicParsing -Headers $headers
|
|
$downloaded = $true
|
|
Write-Host ""Download complete."" -ForegroundColor Green
|
|
break
|
|
}}
|
|
catch {{
|
|
Write-Host (""Download failed from {{0}}: {{1}}"" -f $url, $_.Exception.Message) -ForegroundColor Yellow
|
|
}}
|
|
}}
|
|
|
|
if (-not $downloaded) {{
|
|
throw 'Unable to download SheepIt client from any known URL.'
|
|
}}
|
|
}}
|
|
|
|
[Console]::Error.WriteLine('[SHEEPIT] Starting Java with SheepIt client...')
|
|
Set-Location $sheepDir
|
|
|
|
$javaArgs = @('-XX:+IgnoreUnrecognizedVMOptions', '-jar', $jarPath,
|
|
'-ui', 'text', '--log-stdout', '--verbose',
|
|
'-gpu', '{sheepIt.Gpu}', '-login', '{safeUser}', '-password', '{safeKey}')
|
|
|
|
try {{
|
|
& java @javaArgs
|
|
}}
|
|
catch {{
|
|
Write-Host ('Java execution error: {{0}}' -f $_.Exception.Message) -ForegroundColor Red
|
|
Write-Host ""If the error persists, try reinstalling Java (Temurin 21 recommended)."" -ForegroundColor Yellow
|
|
[Console]::Error.WriteLine(""Java execution error: $($_.Exception.Message)"")
|
|
throw
|
|
}}
|
|
}}
|
|
catch {{
|
|
$errorMsg = ('Error: {{0}}' -f $_.Exception.Message)
|
|
$stackMsg = ('Stack trace: {{0}}' -f $_.ScriptStackTrace)
|
|
Write-Host $errorMsg -ForegroundColor Red
|
|
Write-Host $stackMsg -ForegroundColor DarkRed
|
|
[Console]::Error.WriteLine($errorMsg)
|
|
[Console]::Error.WriteLine($stackMsg)
|
|
exit 1
|
|
}}
|
|
";
|
|
}
|
|
|
|
public string GenerateFlamencoPayload(WorkerConfig worker)
|
|
{
|
|
if (worker.WorkerTypes.Flamenco == null)
|
|
throw new InvalidOperationException("Worker does not have Flamenco configuration");
|
|
|
|
var config = _configService.Load();
|
|
var flamenco = worker.WorkerTypes.Flamenco;
|
|
var globalSettings = config.GlobalSettings;
|
|
var workerPath = globalSettings.FlamencoWorkerPath.Replace("'", "''");
|
|
var drives = string.Join(", ", Array.ConvertAll(flamenco.NetworkDrives.ToArray(), d => $"'{d}'"));
|
|
var paths = string.Join(", ", Array.ConvertAll(flamenco.NetworkPaths.ToArray(), p => $"'{p.Replace("\\", "\\\\")}'"));
|
|
|
|
return $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
# Write startup message to stderr so controller can capture it
|
|
[Console]::Error.WriteLine('[FLAMENCO] Payload script starting...')
|
|
|
|
try {{
|
|
Write-Host ""Setting up network connections..."" -ForegroundColor Cyan
|
|
|
|
$drives = @({drives})
|
|
$networkPaths = @({paths})
|
|
|
|
# Disconnect all existing connections
|
|
Write-Host ""Disconnecting existing network connections..."" -ForegroundColor Yellow
|
|
foreach ($path in $networkPaths) {{ net use $path /delete /y 2>$null }}
|
|
foreach ($drive in $drives) {{ net use $drive /delete /y 2>$null }}
|
|
Write-Host ""All network connections cleared."" -ForegroundColor Green
|
|
|
|
# Connect to network shares (simplified - credentials should be stored securely)
|
|
Write-Host ""Establishing network connections..."" -ForegroundColor Cyan
|
|
# TODO: Add credential handling for network shares
|
|
|
|
# Start worker
|
|
Write-Host ""Starting Flamenco worker..."" -ForegroundColor Cyan
|
|
Set-Location '{workerPath}'
|
|
if (Test-Path 'flamenco-worker.exe') {{
|
|
Write-Host ""Running flamenco-worker.exe..."" -ForegroundColor Green
|
|
$workerProcess = Start-Process -FilePath '.\flamenco-worker.exe' -NoNewWindow -PassThru -Wait
|
|
$exitCode = $workerProcess.ExitCode
|
|
Write-Host ""Flamenco worker process has terminated with exit code: $exitCode"" -ForegroundColor Yellow
|
|
exit $exitCode
|
|
}} else {{
|
|
Write-Host ""Error: flamenco-worker.exe not found in {workerPath}"" -ForegroundColor Red
|
|
[Console]::Error.WriteLine(""Error: flamenco-worker.exe not found in {workerPath}"")
|
|
exit 1
|
|
}}
|
|
}}
|
|
catch {{
|
|
$errorMsg = ('Error: {{0}}' -f $_.Exception.Message)
|
|
$stackMsg = ('Stack trace: {{0}}' -f $_.ScriptStackTrace)
|
|
Write-Host $errorMsg -ForegroundColor Red
|
|
Write-Host $stackMsg -ForegroundColor DarkRed
|
|
[Console]::Error.WriteLine($errorMsg)
|
|
[Console]::Error.WriteLine($stackMsg)
|
|
exit 1
|
|
}}
|
|
";
|
|
}
|
|
|
|
public async Task StartWorkerAsync(WorkerConfig worker, string workerType)
|
|
{
|
|
await DeployControllerAsync(worker);
|
|
|
|
string payloadScript;
|
|
if (workerType == "sheepit")
|
|
{
|
|
payloadScript = GenerateSheepItPayload(worker);
|
|
}
|
|
else if (workerType == "flamenco")
|
|
{
|
|
payloadScript = GenerateFlamencoPayload(worker);
|
|
}
|
|
else
|
|
{
|
|
throw new ArgumentException($"Unknown worker type: {workerType}", nameof(workerType));
|
|
}
|
|
|
|
var payloadBase64 = Convert.ToBase64String(Encoding.Unicode.GetBytes(payloadScript));
|
|
var remoteBasePath = await _sshService.GetWorkerBasePathAsync(worker);
|
|
|
|
var jsonParams = $@"{{""WorkerName"":""{worker.Name}"",""WorkerType"":""{workerType}"",""PayloadBase64"":""{payloadBase64}""}}";
|
|
var jsonParamsBase64 = Convert.ToBase64String(Encoding.Unicode.GetBytes(jsonParams));
|
|
|
|
var ensureScript = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$params = ConvertFrom-Json ([Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('{jsonParamsBase64}')))
|
|
$workerName = $params.WorkerName
|
|
$workerType = $params.WorkerType
|
|
$payloadBase64 = $params.PayloadBase64
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
$instanceRoot = Join-Path (Join-Path $dataRoot $workerType) $workerName
|
|
$logsRoot = Join-Path $instanceRoot 'logs'
|
|
$stateRoot = Join-Path $instanceRoot 'state'
|
|
New-Item -ItemType Directory -Path $logsRoot -Force | Out-Null
|
|
New-Item -ItemType Directory -Path $stateRoot -Force | Out-Null
|
|
$logPath = Join-Path $logsRoot 'worker.log'
|
|
$commandPath = Join-Path $stateRoot 'commands.txt'
|
|
$payloadPath = Join-Path $stateRoot 'payload.ps1'
|
|
$payloadBase64Path = Join-Path $stateRoot 'payload.b64'
|
|
if (-not (Test-Path $logPath)) {{ New-Item -Path $logPath -ItemType File -Force | Out-Null }}
|
|
if (-not (Test-Path $commandPath)) {{ New-Item -Path $commandPath -ItemType File -Force | Out-Null }}
|
|
[IO.File]::WriteAllText($payloadBase64Path, $payloadBase64, [System.Text.Encoding]::UTF8)
|
|
$metaPath = Join-Path $instanceRoot 'state\worker-info.json'
|
|
$controllerPath = Join-Path $dataRoot 'controller.ps1'
|
|
|
|
if (-not (Test-Path $controllerPath)) {{
|
|
throw ""Controller missing at $controllerPath""
|
|
}}
|
|
|
|
$shouldStart = $true
|
|
if (Test-Path $metaPath) {{
|
|
try {{
|
|
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
|
|
if ($meta.Status -eq 'running' -and $meta.WorkerPid) {{
|
|
if (Get-Process -Id $meta.WorkerPid -ErrorAction SilentlyContinue) {{
|
|
Write-Host ""Worker $workerName already running (PID $($meta.WorkerPid)).""
|
|
$shouldStart = $false
|
|
}}
|
|
}}
|
|
}} catch {{
|
|
Write-Host ""Failed to read metadata. Controller will restart worker."" -ForegroundColor Yellow
|
|
}}
|
|
}}
|
|
|
|
if ($shouldStart) {{
|
|
$initialMeta = [pscustomobject]@{{
|
|
WorkerName = $workerName
|
|
WorkerType = $workerType
|
|
Status = 'launching'
|
|
ControllerPid = $null
|
|
WorkerPid = $null
|
|
Restarts = 0
|
|
LastExitCode = $null
|
|
LogPath = $logPath
|
|
CommandPath = $commandPath
|
|
PayloadPath = $payloadPath
|
|
UpdatedAtUtc = (Get-Date).ToUniversalTime()
|
|
}} | ConvertTo-Json -Depth 5
|
|
$initialMeta | Set-Content -Path $metaPath -Encoding UTF8
|
|
|
|
$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
|
|
if ($pwsh) {{
|
|
$psExe = $pwsh.Source
|
|
}}
|
|
else {{
|
|
$psExe = (Get-Command powershell -ErrorAction Stop).Source
|
|
}}
|
|
|
|
$controllerArgs = @(
|
|
'-NoLogo','-NoProfile','-ExecutionPolicy','Bypass',
|
|
'-File',""$controllerPath"",
|
|
'-WorkerName',""$workerName"",
|
|
'-WorkerType',""$workerType"",
|
|
'-PayloadBase64Path',""$payloadBase64Path""
|
|
)
|
|
|
|
Write-Host ""Attempting to start controller with: $psExe"" -ForegroundColor Cyan
|
|
Write-Host ""Controller path: $controllerPath"" -ForegroundColor Cyan
|
|
Write-Host ""Arguments: $($controllerArgs -join ' ')"" -ForegroundColor Cyan
|
|
|
|
try {{
|
|
$controllerProcess = Start-Process -FilePath $psExe -ArgumentList $controllerArgs -WindowStyle Hidden -PassThru -ErrorAction Stop
|
|
if ($controllerProcess) {{
|
|
$controllerPid = $controllerProcess.Id
|
|
Write-Host ""Controller process started with PID: $controllerPid"" -ForegroundColor Green
|
|
|
|
# Give it a moment to initialize
|
|
Start-Sleep -Milliseconds 500
|
|
|
|
# Verify process is still running
|
|
$stillRunning = Get-Process -Id $controllerPid -ErrorAction SilentlyContinue
|
|
if (-not $stillRunning) {{
|
|
throw ""Controller process (PID: $controllerPid) exited immediately after start""
|
|
}}
|
|
|
|
# Update metadata with controller PID
|
|
$updatedMeta = [pscustomobject]@{{
|
|
WorkerName = $workerName
|
|
WorkerType = $workerType
|
|
Status = 'launching'
|
|
ControllerPid = $controllerPid
|
|
WorkerPid = $null
|
|
Restarts = 0
|
|
LastExitCode = $null
|
|
LogPath = $logPath
|
|
CommandPath = $commandPath
|
|
PayloadPath = $payloadPath
|
|
UpdatedAtUtc = (Get-Date).ToUniversalTime()
|
|
}} | ConvertTo-Json -Depth 5
|
|
$updatedMeta | Set-Content -Path $metaPath -Encoding UTF8
|
|
|
|
# Ensure file is flushed to disk before controller reads it
|
|
Start-Sleep -Milliseconds 200
|
|
|
|
Write-Host ""Worker $workerName started under controller (PID: $controllerPid)."" -ForegroundColor Green
|
|
}} else {{
|
|
throw ""Start-Process returned null - controller failed to start""
|
|
}}
|
|
}} catch {{
|
|
$errorMsg = $_.Exception.Message
|
|
$errorDetails = $_.Exception.ToString()
|
|
Write-Host ""Failed to start controller: $errorMsg"" -ForegroundColor Red
|
|
Write-Host ""Error details: $errorDetails"" -ForegroundColor Red
|
|
[Console]::Error.WriteLine(""Controller startup error: $errorMsg"")
|
|
[Console]::Error.WriteLine(""Full error: $errorDetails"")
|
|
|
|
# Update metadata with error
|
|
$errorMeta = [pscustomobject]@{{
|
|
WorkerName = $workerName
|
|
WorkerType = $workerType
|
|
Status = 'error'
|
|
ControllerPid = $null
|
|
WorkerPid = $null
|
|
Restarts = 0
|
|
LastExitCode = 1
|
|
LogPath = $logPath
|
|
CommandPath = $commandPath
|
|
PayloadPath = $payloadPath
|
|
UpdatedAtUtc = (Get-Date).ToUniversalTime()
|
|
ErrorMessage = ""Controller startup failed: $errorMsg""
|
|
}} | ConvertTo-Json -Depth 5
|
|
$errorMeta | Set-Content -Path $metaPath -Encoding UTF8
|
|
throw
|
|
}}
|
|
}}
|
|
";
|
|
|
|
// Pipe script through stdin to avoid command line length limits
|
|
string? ensureScriptOutput = null;
|
|
try
|
|
{
|
|
ensureScriptOutput = await _sshService.ExecuteRemoteScriptAsync(worker, ensureScript);
|
|
// Log any output from ensureScript for debugging
|
|
if (!string.IsNullOrWhiteSpace(ensureScriptOutput))
|
|
{
|
|
System.Diagnostics.Debug.WriteLine($"EnsureScript output: {ensureScriptOutput}");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// If ensureScript fails, include that in diagnostics
|
|
System.Diagnostics.Debug.WriteLine($"EnsureScript execution error: {ex.Message}");
|
|
// Don't throw here - let the status check below handle it with full diagnostics
|
|
}
|
|
|
|
// Wait a moment for the controller to start and update metadata
|
|
// Give extra time for the metadata write to complete
|
|
await Task.Delay(3000);
|
|
|
|
// Verify the worker actually started by checking metadata
|
|
// Try up to 3 times with delays to account for slower startup
|
|
WorkerStatus? status = null;
|
|
for (int i = 0; i < 3; i++)
|
|
{
|
|
status = await GetWorkerStatusAsync(worker, workerType);
|
|
if (status != null && (status.Status == "running" || status.Status == "launching"))
|
|
{
|
|
break;
|
|
}
|
|
if (i < 2) // Don't delay on last attempt
|
|
{
|
|
await Task.Delay(1000);
|
|
}
|
|
}
|
|
|
|
if (status == null || (status.Status != "running" && status.Status != "launching"))
|
|
{
|
|
// Gather comprehensive diagnostic information
|
|
var diagnostics = await GatherDiagnosticsAsync(worker, workerType);
|
|
var errorMessage = status?.ErrorMessage ?? "Worker failed to start. Check logs for details.";
|
|
var statusText = status?.Status ?? "unknown";
|
|
|
|
// Include ensureScript output if available
|
|
var ensureOutputInfo = string.IsNullOrWhiteSpace(ensureScriptOutput)
|
|
? ""
|
|
: $"\n\n=== EnsureScript Output ===\n{ensureScriptOutput}";
|
|
|
|
throw new InvalidOperationException($"Worker did not start successfully. Status: {statusText}. {errorMessage}\n\n{diagnostics}{ensureOutputInfo}");
|
|
}
|
|
}
|
|
|
|
private async Task<WorkerStatus?> GetWorkerStatusAsync(WorkerConfig worker, string workerType)
|
|
{
|
|
var remoteBasePath = await _sshService.GetWorkerBasePathAsync(worker);
|
|
var instanceRoot = Path.Combine(remoteBasePath, workerType, worker.Name);
|
|
var metaPath = Path.Combine(instanceRoot, "state", "worker-info.json");
|
|
|
|
var script = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
$instanceRoot = Join-Path (Join-Path $dataRoot '{workerType}') '{worker.Name}'
|
|
$metaPath = Join-Path $instanceRoot 'state\worker-info.json'
|
|
if (Test-Path $metaPath) {{
|
|
try {{
|
|
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
|
|
$meta | ConvertTo-Json -Depth 5 -Compress
|
|
}} catch {{
|
|
Write-Error ""Error reading metadata: $($_.Exception.Message)""
|
|
}}
|
|
}} else {{
|
|
Write-Error ""Metadata file not found at $metaPath""
|
|
}}
|
|
";
|
|
|
|
try
|
|
{
|
|
var output = await _sshService.ExecuteRemoteScriptAsync(worker, script);
|
|
if (string.IsNullOrWhiteSpace(output))
|
|
return null;
|
|
|
|
var lines = output.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
|
// Look for JSON line - it might be on a single line or multiple lines
|
|
var jsonLines = lines.Where(line => line.Trim().StartsWith("{")).ToList();
|
|
|
|
string? jsonText = null;
|
|
if (jsonLines.Count > 0)
|
|
{
|
|
// Use the first JSON line found
|
|
jsonText = jsonLines[0].Trim();
|
|
}
|
|
else
|
|
{
|
|
// Try to find JSON that might span multiple lines
|
|
var fullText = string.Join("", lines);
|
|
var startIdx = fullText.IndexOf('{');
|
|
if (startIdx >= 0)
|
|
{
|
|
var endIdx = fullText.LastIndexOf('}');
|
|
if (endIdx > startIdx)
|
|
{
|
|
jsonText = fullText.Substring(startIdx, endIdx - startIdx + 1);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(jsonText))
|
|
return null;
|
|
|
|
var options = new JsonSerializerOptions
|
|
{
|
|
PropertyNameCaseInsensitive = true
|
|
};
|
|
|
|
return JsonSerializer.Deserialize<WorkerStatus>(jsonText, options);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Log the exception for debugging but return null
|
|
System.Diagnostics.Debug.WriteLine($"Error parsing worker status: {ex.Message}");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private async Task<string> GatherDiagnosticsAsync(WorkerConfig worker, string workerType)
|
|
{
|
|
var diagnostics = new StringBuilder();
|
|
|
|
var script = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$ErrorActionPreference = 'Continue'
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
$instanceRoot = Join-Path (Join-Path $dataRoot '{workerType}') '{worker.Name}'
|
|
$metaPath = Join-Path $instanceRoot 'state\worker-info.json'
|
|
$logPath = Join-Path $instanceRoot 'logs\worker.log'
|
|
$controllerPath = Join-Path $dataRoot 'controller.ps1'
|
|
|
|
Write-Host ""=== Diagnostics ==="" -ForegroundColor Cyan
|
|
Write-Host ""Metadata file exists: $((Test-Path $metaPath))""
|
|
Write-Host ""Log file exists: $((Test-Path $logPath))""
|
|
Write-Host ""Controller script exists: $((Test-Path $controllerPath))""
|
|
|
|
# Check controller script content
|
|
if (Test-Path $controllerPath) {{
|
|
$controllerSize = (Get-Item $controllerPath).Length
|
|
Write-Host ""Controller script size: $controllerSize bytes""
|
|
$firstLine = Get-Content $controllerPath -First 1 -ErrorAction SilentlyContinue
|
|
if ($firstLine) {{
|
|
Write-Host ""Controller script first line: $firstLine""
|
|
}}
|
|
|
|
# Try to test if controller script can be parsed/executed
|
|
Write-Host ""`n=== Testing Controller Script ==="" -ForegroundColor Cyan
|
|
try {{
|
|
$testResult = powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command ""& {{ Get-Command -Name '$controllerPath' -ErrorAction Stop }}"" 2>&1
|
|
Write-Host ""Controller script syntax check: Passed""
|
|
}} catch {{
|
|
Write-Host ""Controller script syntax check failed: $($_.Exception.Message)"" -ForegroundColor Red
|
|
}}
|
|
|
|
# Check if we can manually invoke the controller with test parameters
|
|
$testParams = @('-WorkerName', 'TEST', '-WorkerType', 'sheepit', '-PayloadBase64Path', 'C:\temp\test.b64')
|
|
Write-Host ""Attempting to test controller invocation...""
|
|
try {{
|
|
$testProcess = Start-Process -FilePath powershell -ArgumentList @('-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', ""$controllerPath"", '-WorkerName', 'TEST') -PassThru -WindowStyle Hidden -ErrorAction Stop
|
|
Start-Sleep -Milliseconds 200
|
|
if ($testProcess.HasExited) {{
|
|
Write-Host ""Test controller process exited with code: $($testProcess.ExitCode)"" -ForegroundColor Yellow
|
|
}} else {{
|
|
Write-Host ""Test controller process started successfully (PID: $($testProcess.Id))"" -ForegroundColor Green
|
|
Stop-Process -Id $testProcess.Id -Force -ErrorAction SilentlyContinue
|
|
}}
|
|
}} catch {{
|
|
Write-Host ""Test controller invocation failed: $($_.Exception.Message)"" -ForegroundColor Red
|
|
Write-Host ""Full error: $($_.Exception.ToString())"" -ForegroundColor Red
|
|
}}
|
|
}}
|
|
|
|
if (Test-Path $metaPath) {{
|
|
try {{
|
|
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
|
|
Write-Host ""Metadata Status: $($meta.Status)""
|
|
Write-Host ""Controller PID: $($meta.ControllerPid)""
|
|
Write-Host ""Worker PID: $($meta.WorkerPid)""
|
|
if ($meta.ErrorMessage) {{
|
|
Write-Host ""Error Message: $($meta.ErrorMessage)"" -ForegroundColor Red
|
|
}}
|
|
}} catch {{
|
|
Write-Host ""Error reading metadata: $($_.Exception.Message)"" -ForegroundColor Red
|
|
Write-Host ""Metadata file content (first 500 chars):"" -ForegroundColor Yellow
|
|
$metaContent = Get-Content $metaPath -Raw -ErrorAction SilentlyContinue
|
|
if ($metaContent) {{
|
|
Write-Host $metaContent.Substring(0, [Math]::Min(500, $metaContent.Length))
|
|
}}
|
|
}}
|
|
}} else {{
|
|
Write-Host ""Metadata file not found - controller may not have started"" -ForegroundColor Yellow
|
|
}}
|
|
|
|
# Check if controller process is running
|
|
$controllerRunning = $false
|
|
$controllerProcessInfo = ""N/A""
|
|
if (Test-Path $metaPath) {{
|
|
try {{
|
|
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
|
|
if ($meta.ControllerPid) {{
|
|
$proc = Get-Process -Id $meta.ControllerPid -ErrorAction SilentlyContinue
|
|
$controllerRunning = ($null -ne $proc)
|
|
if ($controllerRunning) {{
|
|
$controllerProcessInfo = ""Running (PID: $($meta.ControllerPid))""
|
|
}} else {{
|
|
$controllerProcessInfo = ""Not running (PID: $($meta.ControllerPid) - process not found)""
|
|
}}
|
|
}} else {{
|
|
$controllerProcessInfo = ""No controller PID in metadata""
|
|
}}
|
|
}} catch {{
|
|
$controllerProcessInfo = ""Error checking: $($_.Exception.Message)""
|
|
}}
|
|
}} else {{
|
|
$controllerProcessInfo = ""Metadata file not found""
|
|
}}
|
|
Write-Host ""Controller process: $controllerProcessInfo""
|
|
|
|
Write-Host ""`n=== Recent Log Entries (last 30 lines) ==="" -ForegroundColor Cyan
|
|
if (Test-Path $logPath) {{
|
|
try {{
|
|
$logFileInfo = Get-Item $logPath -ErrorAction SilentlyContinue
|
|
if ($logFileInfo) {{
|
|
Write-Host ""Log file size: $($logFileInfo.Length) bytes"" -ForegroundColor Gray
|
|
}}
|
|
|
|
$logContent = Get-Content $logPath -ErrorAction SilentlyContinue
|
|
if ($logContent) {{
|
|
$logLines = if ($logContent.Count -gt 30) {{ $logContent[-30..-1] }} else {{ $logContent }}
|
|
Write-Host ""Found $($logLines.Count) log lines (showing last $([Math]::Min(30, $logLines.Count)))""
|
|
$logLines | ForEach-Object {{ Write-Host $_ }}
|
|
}} else {{
|
|
Write-Host ""Log file exists but appears to be empty or unreadable"" -ForegroundColor Yellow
|
|
Write-Host ""Attempting to read raw bytes..."" -ForegroundColor Gray
|
|
try {{
|
|
$rawBytes = [IO.File]::ReadAllBytes($logPath)
|
|
Write-Host ""Log file contains $($rawBytes.Length) bytes""
|
|
if ($rawBytes.Length -gt 0) {{
|
|
$text = [System.Text.Encoding]::UTF8.GetString($rawBytes)
|
|
Write-Host ""Log content (first 500 chars):"" -ForegroundColor Cyan
|
|
Write-Host $text.Substring(0, [Math]::Min(500, $text.Length))
|
|
}}
|
|
}} catch {{
|
|
Write-Host ""Could not read log file bytes: $($_.Exception.Message)"" -ForegroundColor Red
|
|
}}
|
|
}}
|
|
}} catch {{
|
|
Write-Host ""Error reading log: $($_.Exception.Message)"" -ForegroundColor Red
|
|
Write-Host ""Stack trace: $($_.ScriptStackTrace)"" -ForegroundColor DarkRed
|
|
}}
|
|
}} else {{
|
|
Write-Host ""Log file does not exist yet at: $logPath"" -ForegroundColor Yellow
|
|
}}
|
|
";
|
|
|
|
try
|
|
{
|
|
var output = await _sshService.ExecuteRemoteScriptAsync(worker, script);
|
|
return output.Trim();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return $"Unable to gather diagnostics: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
private class WorkerStatus
|
|
{
|
|
public string? Status { get; set; }
|
|
public int? WorkerPid { get; set; }
|
|
public string? ErrorMessage { get; set; }
|
|
}
|
|
|
|
public async Task StopWorkerAsync(WorkerConfig worker, string workerType)
|
|
{
|
|
var script = $@"
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
|
|
$instanceRoot = Join-Path (Join-Path $dataRoot '{workerType}') '{worker.Name}'
|
|
$commandPath = Join-Path $instanceRoot 'state\commands.txt'
|
|
[IO.File]::WriteAllText($commandPath, 'quit', [System.Text.Encoding]::UTF8)
|
|
Write-Host ""Quit command sent to worker {worker.Name}.""
|
|
";
|
|
|
|
await _sshService.ExecuteRemoteCommandAsync(worker, $"powershell -NoLogo -NoProfile -NonInteractive -OutputFormat Text -ExecutionPolicy Bypass -EncodedCommand {Convert.ToBase64String(Encoding.Unicode.GetBytes(script))}");
|
|
}
|
|
}
|
|
}
|
|
|