launch error display

This commit is contained in:
Nathan
2025-12-17 17:18:27 -07:00
parent 84a518a9c9
commit 188d74c072
7 changed files with 21351 additions and 19 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -224,8 +224,24 @@ function Get-PendingCommands {
# endregion # endregion
try { try {
# Check if metadata already exists with a ControllerPid (set by ensureScript)
$existingControllerPid = $null
if (Test-Path $metaPath) {
try {
$existing = Get-Content $metaPath -Raw | ConvertFrom-Json
if ($existing.ControllerPid) {
$existingControllerPid = $existing.ControllerPid
}
} catch {
# Ignore parse errors
}
}
# Use existing ControllerPid if available, otherwise use current PID
$controllerPidToUse = if ($existingControllerPid) { $existingControllerPid } else { $PID }
# record initial state before launching worker # record initial state before launching worker
Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $PID -Restarts 0 Write-Metadata -Status 'initializing' -WorkerPid $null -ControllerPid $controllerPidToUse -Restarts 0
$resolvedPayloadBase64 = Resolve-PayloadBase64 $resolvedPayloadBase64 = Resolve-PayloadBase64
$PayloadBase64 = $resolvedPayloadBase64 $PayloadBase64 = $resolvedPayloadBase64

View File

@@ -275,10 +275,28 @@ namespace UnifiedFarmLauncher.Services
if (process.ExitCode != 0 && !interactive) if (process.ExitCode != 0 && !interactive)
{ {
throw new InvalidOperationException($"SSH script execution failed with exit code {process.ExitCode}: {error}"); var errorText = error.ToString();
var combinedOutput = output.ToString();
if (!string.IsNullOrWhiteSpace(errorText))
{
combinedOutput = string.IsNullOrWhiteSpace(combinedOutput)
? errorText
: $"{combinedOutput}\n\n=== STDERR ===\n{errorText}";
}
throw new InvalidOperationException($"SSH script execution failed with exit code {process.ExitCode}: {combinedOutput}");
} }
return output.ToString(); // Include stderr in output if present (for diagnostics)
var result = output.ToString();
var stderrText = error.ToString();
if (!string.IsNullOrWhiteSpace(stderrText) && !interactive)
{
result = string.IsNullOrWhiteSpace(result)
? stderrText
: $"{result}\n\n=== STDERR ===\n{stderrText}";
}
return result;
} }
public async Task<string> GetWorkerBasePathAsync(WorkerConfig worker) public async Task<string> GetWorkerBasePathAsync(WorkerConfig worker)

View File

@@ -1,7 +1,10 @@
using System; using System;
using System.IO; using System.IO;
using System.Linq;
using System.Reflection; using System.Reflection;
using System.Text; using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using UnifiedFarmLauncher.Models; using UnifiedFarmLauncher.Models;
@@ -325,13 +328,356 @@ if ($shouldStart) {{
'-PayloadBase64Path',""$payloadBase64Path"" '-PayloadBase64Path',""$payloadBase64Path""
) )
Start-Process -FilePath $psExe -ArgumentList $controllerArgs -WindowStyle Hidden | Out-Null Write-Host ""Attempting to start controller with: $psExe"" -ForegroundColor Cyan
Write-Host ""Worker $workerName started under controller."" -ForegroundColor Green Write-Host ""Controller path: $controllerPath"" -ForegroundColor Cyan
Write-Host ""Arguments: $($controllerArgs -join ' ')"" -ForegroundColor Cyan
try {{
$controllerProcess = Start-Process -FilePath $psExe -ArgumentList $controllerArgs -WindowStyle Hidden -PassThru -ErrorAction Stop
if ($controllerProcess) {{
$controllerPid = $controllerProcess.Id
Write-Host ""Controller process started with PID: $controllerPid"" -ForegroundColor Green
# Give it a moment to initialize
Start-Sleep -Milliseconds 500
# Verify process is still running
$stillRunning = Get-Process -Id $controllerPid -ErrorAction SilentlyContinue
if (-not $stillRunning) {{
throw ""Controller process (PID: $controllerPid) exited immediately after start""
}}
# Update metadata with controller PID
$updatedMeta = [pscustomobject]@{{
WorkerName = $workerName
WorkerType = $workerType
Status = 'launching'
ControllerPid = $controllerPid
WorkerPid = $null
Restarts = 0
LastExitCode = $null
LogPath = $logPath
CommandPath = $commandPath
PayloadPath = $payloadPath
UpdatedAtUtc = (Get-Date).ToUniversalTime()
}} | ConvertTo-Json -Depth 5
$updatedMeta | Set-Content -Path $metaPath -Encoding UTF8
# Ensure file is flushed to disk before controller reads it
Start-Sleep -Milliseconds 200
Write-Host ""Worker $workerName started under controller (PID: $controllerPid)."" -ForegroundColor Green
}} else {{
throw ""Start-Process returned null - controller failed to start""
}}
}} catch {{
$errorMsg = $_.Exception.Message
$errorDetails = $_.Exception.ToString()
Write-Host ""Failed to start controller: $errorMsg"" -ForegroundColor Red
Write-Host ""Error details: $errorDetails"" -ForegroundColor Red
[Console]::Error.WriteLine(""Controller startup error: $errorMsg"")
[Console]::Error.WriteLine(""Full error: $errorDetails"")
# Update metadata with error
$errorMeta = [pscustomobject]@{{
WorkerName = $workerName
WorkerType = $workerType
Status = 'error'
ControllerPid = $null
WorkerPid = $null
Restarts = 0
LastExitCode = 1
LogPath = $logPath
CommandPath = $commandPath
PayloadPath = $payloadPath
UpdatedAtUtc = (Get-Date).ToUniversalTime()
ErrorMessage = ""Controller startup failed: $errorMsg""
}} | ConvertTo-Json -Depth 5
$errorMeta | Set-Content -Path $metaPath -Encoding UTF8
throw
}}
}} }}
"; ";
// Pipe script through stdin to avoid command line length limits // Pipe script through stdin to avoid command line length limits
await _sshService.ExecuteRemoteScriptAsync(worker, ensureScript); string? ensureScriptOutput = null;
try
{
ensureScriptOutput = await _sshService.ExecuteRemoteScriptAsync(worker, ensureScript);
// Log any output from ensureScript for debugging
if (!string.IsNullOrWhiteSpace(ensureScriptOutput))
{
System.Diagnostics.Debug.WriteLine($"EnsureScript output: {ensureScriptOutput}");
}
}
catch (Exception ex)
{
// If ensureScript fails, include that in diagnostics
System.Diagnostics.Debug.WriteLine($"EnsureScript execution error: {ex.Message}");
// Don't throw here - let the status check below handle it with full diagnostics
}
// Wait a moment for the controller to start and update metadata
// Give extra time for the metadata write to complete
await Task.Delay(3000);
// Verify the worker actually started by checking metadata
// Try up to 3 times with delays to account for slower startup
WorkerStatus? status = null;
for (int i = 0; i < 3; i++)
{
status = await GetWorkerStatusAsync(worker, workerType);
if (status != null && (status.Status == "running" || status.Status == "launching"))
{
break;
}
if (i < 2) // Don't delay on last attempt
{
await Task.Delay(1000);
}
}
if (status == null || (status.Status != "running" && status.Status != "launching"))
{
// Gather comprehensive diagnostic information
var diagnostics = await GatherDiagnosticsAsync(worker, workerType);
var errorMessage = status?.ErrorMessage ?? "Worker failed to start. Check logs for details.";
var statusText = status?.Status ?? "unknown";
// Include ensureScript output if available
var ensureOutputInfo = string.IsNullOrWhiteSpace(ensureScriptOutput)
? ""
: $"\n\n=== EnsureScript Output ===\n{ensureScriptOutput}";
throw new InvalidOperationException($"Worker did not start successfully. Status: {statusText}. {errorMessage}\n\n{diagnostics}{ensureOutputInfo}");
}
}
private async Task<WorkerStatus?> GetWorkerStatusAsync(WorkerConfig worker, string workerType)
{
var remoteBasePath = await _sshService.GetWorkerBasePathAsync(worker);
var instanceRoot = Path.Combine(remoteBasePath, workerType, worker.Name);
var metaPath = Path.Combine(instanceRoot, "state", "worker-info.json");
var script = $@"
$ProgressPreference = 'SilentlyContinue'
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
$instanceRoot = Join-Path (Join-Path $dataRoot '{workerType}') '{worker.Name}'
$metaPath = Join-Path $instanceRoot 'state\worker-info.json'
if (Test-Path $metaPath) {{
try {{
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
$meta | ConvertTo-Json -Depth 5 -Compress
}} catch {{
Write-Error ""Error reading metadata: $($_.Exception.Message)""
}}
}} else {{
Write-Error ""Metadata file not found at $metaPath""
}}
";
try
{
var output = await _sshService.ExecuteRemoteScriptAsync(worker, script);
if (string.IsNullOrWhiteSpace(output))
return null;
var lines = output.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
// Look for JSON line - it might be on a single line or multiple lines
var jsonLines = lines.Where(line => line.Trim().StartsWith("{")).ToList();
string? jsonText = null;
if (jsonLines.Count > 0)
{
// Use the first JSON line found
jsonText = jsonLines[0].Trim();
}
else
{
// Try to find JSON that might span multiple lines
var fullText = string.Join("", lines);
var startIdx = fullText.IndexOf('{');
if (startIdx >= 0)
{
var endIdx = fullText.LastIndexOf('}');
if (endIdx > startIdx)
{
jsonText = fullText.Substring(startIdx, endIdx - startIdx + 1);
}
}
}
if (string.IsNullOrEmpty(jsonText))
return null;
var options = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
};
return JsonSerializer.Deserialize<WorkerStatus>(jsonText, options);
}
catch (Exception ex)
{
// Log the exception for debugging but return null
System.Diagnostics.Debug.WriteLine($"Error parsing worker status: {ex.Message}");
return null;
}
}
private async Task<string> GatherDiagnosticsAsync(WorkerConfig worker, string workerType)
{
var diagnostics = new StringBuilder();
var script = $@"
$ProgressPreference = 'SilentlyContinue'
$ErrorActionPreference = 'Continue'
$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
$instanceRoot = Join-Path (Join-Path $dataRoot '{workerType}') '{worker.Name}'
$metaPath = Join-Path $instanceRoot 'state\worker-info.json'
$logPath = Join-Path $instanceRoot 'logs\worker.log'
$controllerPath = Join-Path $dataRoot 'controller.ps1'
Write-Host ""=== Diagnostics ==="" -ForegroundColor Cyan
Write-Host ""Metadata file exists: $((Test-Path $metaPath))""
Write-Host ""Log file exists: $((Test-Path $logPath))""
Write-Host ""Controller script exists: $((Test-Path $controllerPath))""
# Check controller script content
if (Test-Path $controllerPath) {{
$controllerSize = (Get-Item $controllerPath).Length
Write-Host ""Controller script size: $controllerSize bytes""
$firstLine = Get-Content $controllerPath -First 1 -ErrorAction SilentlyContinue
if ($firstLine) {{
Write-Host ""Controller script first line: $firstLine""
}}
# Try to test if controller script can be parsed/executed
Write-Host ""`n=== Testing Controller Script ==="" -ForegroundColor Cyan
try {{
$testResult = powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command ""& {{ Get-Command -Name '$controllerPath' -ErrorAction Stop }}"" 2>&1
Write-Host ""Controller script syntax check: Passed""
}} catch {{
Write-Host ""Controller script syntax check failed: $($_.Exception.Message)"" -ForegroundColor Red
}}
# Check if we can manually invoke the controller with test parameters
$testParams = @('-WorkerName', 'TEST', '-WorkerType', 'sheepit', '-PayloadBase64Path', 'C:\temp\test.b64')
Write-Host ""Attempting to test controller invocation...""
try {{
$testProcess = Start-Process -FilePath powershell -ArgumentList @('-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', ""$controllerPath"", '-WorkerName', 'TEST') -PassThru -WindowStyle Hidden -ErrorAction Stop
Start-Sleep -Milliseconds 200
if ($testProcess.HasExited) {{
Write-Host ""Test controller process exited with code: $($testProcess.ExitCode)"" -ForegroundColor Yellow
}} else {{
Write-Host ""Test controller process started successfully (PID: $($testProcess.Id))"" -ForegroundColor Green
Stop-Process -Id $testProcess.Id -Force -ErrorAction SilentlyContinue
}}
}} catch {{
Write-Host ""Test controller invocation failed: $($_.Exception.Message)"" -ForegroundColor Red
Write-Host ""Full error: $($_.Exception.ToString())"" -ForegroundColor Red
}}
}}
if (Test-Path $metaPath) {{
try {{
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
Write-Host ""Metadata Status: $($meta.Status)""
Write-Host ""Controller PID: $($meta.ControllerPid)""
Write-Host ""Worker PID: $($meta.WorkerPid)""
if ($meta.ErrorMessage) {{
Write-Host ""Error Message: $($meta.ErrorMessage)"" -ForegroundColor Red
}}
}} catch {{
Write-Host ""Error reading metadata: $($_.Exception.Message)"" -ForegroundColor Red
Write-Host ""Metadata file content (first 500 chars):"" -ForegroundColor Yellow
$metaContent = Get-Content $metaPath -Raw -ErrorAction SilentlyContinue
if ($metaContent) {{
Write-Host $metaContent.Substring(0, [Math]::Min(500, $metaContent.Length))
}}
}}
}} else {{
Write-Host ""Metadata file not found - controller may not have started"" -ForegroundColor Yellow
}}
# Check if controller process is running
$controllerRunning = $false
$controllerProcessInfo = ""N/A""
if (Test-Path $metaPath) {{
try {{
$meta = Get-Content $metaPath -Raw | ConvertFrom-Json
if ($meta.ControllerPid) {{
$proc = Get-Process -Id $meta.ControllerPid -ErrorAction SilentlyContinue
$controllerRunning = ($null -ne $proc)
if ($controllerRunning) {{
$controllerProcessInfo = ""Running (PID: $($meta.ControllerPid))""
}} else {{
$controllerProcessInfo = ""Not running (PID: $($meta.ControllerPid) - process not found)""
}}
}} else {{
$controllerProcessInfo = ""No controller PID in metadata""
}}
}} catch {{
$controllerProcessInfo = ""Error checking: $($_.Exception.Message)""
}}
}} else {{
$controllerProcessInfo = ""Metadata file not found""
}}
Write-Host ""Controller process: $controllerProcessInfo""
Write-Host ""`n=== Recent Log Entries (last 30 lines) ==="" -ForegroundColor Cyan
if (Test-Path $logPath) {{
try {{
$logFileInfo = Get-Item $logPath -ErrorAction SilentlyContinue
if ($logFileInfo) {{
Write-Host ""Log file size: $($logFileInfo.Length) bytes"" -ForegroundColor Gray
}}
$logContent = Get-Content $logPath -ErrorAction SilentlyContinue
if ($logContent) {{
$logLines = if ($logContent.Count -gt 30) {{ $logContent[-30..-1] }} else {{ $logContent }}
Write-Host ""Found $($logLines.Count) log lines (showing last $([Math]::Min(30, $logLines.Count)))""
$logLines | ForEach-Object {{ Write-Host $_ }}
}} else {{
Write-Host ""Log file exists but appears to be empty or unreadable"" -ForegroundColor Yellow
Write-Host ""Attempting to read raw bytes..."" -ForegroundColor Gray
try {{
$rawBytes = [IO.File]::ReadAllBytes($logPath)
Write-Host ""Log file contains $($rawBytes.Length) bytes""
if ($rawBytes.Length -gt 0) {{
$text = [System.Text.Encoding]::UTF8.GetString($rawBytes)
Write-Host ""Log content (first 500 chars):"" -ForegroundColor Cyan
Write-Host $text.Substring(0, [Math]::Min(500, $text.Length))
}}
}} catch {{
Write-Host ""Could not read log file bytes: $($_.Exception.Message)"" -ForegroundColor Red
}}
}}
}} catch {{
Write-Host ""Error reading log: $($_.Exception.Message)"" -ForegroundColor Red
Write-Host ""Stack trace: $($_.ScriptStackTrace)"" -ForegroundColor DarkRed
}}
}} else {{
Write-Host ""Log file does not exist yet at: $logPath"" -ForegroundColor Yellow
}}
";
try
{
var output = await _sshService.ExecuteRemoteScriptAsync(worker, script);
return output.Trim();
}
catch (Exception ex)
{
return $"Unable to gather diagnostics: {ex.Message}";
}
}
private class WorkerStatus
{
public string? Status { get; set; }
public int? WorkerPid { get; set; }
public string? ErrorMessage { get; set; }
} }
public async Task StopWorkerAsync(WorkerConfig worker, string workerType) public async Task StopWorkerAsync(WorkerConfig worker, string workerType)

37
Views/ErrorDialog.axaml Normal file
View File

@@ -0,0 +1,37 @@
<Window xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
x:Class="UnifiedFarmLauncher.Views.ErrorDialog"
Title="Error"
Width="700"
Height="500"
MinWidth="500"
MinHeight="300"
WindowStartupLocation="CenterOwner">
<Grid RowDefinitions="Auto,*,Auto" Margin="15">
<!-- Header with icon and title -->
<StackPanel Orientation="Horizontal" Grid.Row="0" Margin="0,0,0,15">
<TextBlock Text="❌" FontSize="24" VerticalAlignment="Center" Margin="0,0,10,0"/>
<TextBlock Text="Error" FontSize="18" FontWeight="Bold" VerticalAlignment="Center"/>
</StackPanel>
<!-- Scrollable error content -->
<ScrollViewer Grid.Row="1"
VerticalScrollBarVisibility="Auto"
HorizontalScrollBarVisibility="Disabled">
<Border>
<TextBlock x:Name="ErrorTextBlock"
TextWrapping="Wrap"
FontFamily="Consolas, Courier New, monospace"
FontSize="12"/>
</Border>
</ScrollViewer>
<!-- OK Button -->
<Button Grid.Row="2"
Content="OK"
HorizontalAlignment="Right"
Width="80"
Margin="0,15,0,0"
Click="OkButton_Click"/>
</Grid>
</Window>

View File

@@ -0,0 +1,23 @@
using Avalonia.Controls;
using Avalonia.Interactivity;
namespace UnifiedFarmLauncher.Views
{
public partial class ErrorDialog : Window
{
public ErrorDialog()
{
InitializeComponent();
}
public ErrorDialog(string errorMessage) : this()
{
this.FindControl<TextBlock>("ErrorTextBlock")!.Text = errorMessage;
}
private void OkButton_Click(object? sender, RoutedEventArgs e)
{
Close();
}
}
}

View File

@@ -128,10 +128,8 @@ namespace UnifiedFarmLauncher.Views
} }
catch (System.Exception ex) catch (System.Exception ex)
{ {
var errorBox = MessageBoxManager.GetMessageBoxStandard("Error", var errorDialog = new ErrorDialog($"Failed to start worker: {ex.Message}");
$"Failed to start worker: {ex.Message}", await errorDialog.ShowDialog(this);
ButtonEnum.Ok, MsBox.Avalonia.Enums.Icon.Error);
await errorBox.ShowAsync();
} }
} }
} }
@@ -171,10 +169,8 @@ namespace UnifiedFarmLauncher.Views
} }
catch (System.Exception ex) catch (System.Exception ex)
{ {
var errorBox = MessageBoxManager.GetMessageBoxStandard("Error", var errorDialog = new ErrorDialog($"Failed to stop worker: {ex.Message}");
$"Failed to stop worker: {ex.Message}", await errorDialog.ShowDialog(this);
ButtonEnum.Ok, MsBox.Avalonia.Enums.Icon.Error);
await errorBox.ShowAsync();
} }
} }
} }
@@ -210,10 +206,8 @@ namespace UnifiedFarmLauncher.Views
} }
catch (System.Exception ex) catch (System.Exception ex)
{ {
var errorBox = MessageBoxManager.GetMessageBoxStandard("Error", var errorDialog = new ErrorDialog($"Failed to attach to worker: {ex.Message}");
$"Failed to attach to worker: {ex.Message}", await errorDialog.ShowDialog(this);
ButtonEnum.Ok, MsBox.Avalonia.Enums.Icon.Error);
await errorBox.ShowAsync();
} }
} }
} }