Flamenco-Management/unified_flamenco_launcher.ps1

# Master Unified Flamenco Launcher Script
Write-Host "==========================================" -ForegroundColor Cyan
Write-Host "    UNIFIED FLAMENCO WORKER LAUNCHER" -ForegroundColor Cyan
Write-Host "==========================================" -ForegroundColor Cyan
Write-Host

# Define worker-specific configuration
$workers = @(
    @{
        ID = 1
        Name = "i9kf"
        SSHHost = "i9kf"
        SSHPort = 22
        SSHArgs = "-t i9kf"
    },
    @{
        ID = 2
        Name = "blender-boss"
        SSHHost = "blender-boss"
        SSHPort = 22
        SSHArgs = "-t blender-boss"
    },
    @{
        ID = 3
        Name = "max"
        SSHHost = "max"
        SSHPort = 22
        SSHArgs = "-t max"
    },
    @{
        ID = 4
        Name = "masterbox"
        SSHHost = "masterbox"
        SSHPort = 22
        SSHArgs = "-t masterbox"
    },
    @{
        ID = 5
        Name = "echo"
        SSHHost = "echo"
        SSHPort = 22
        SSHArgs = "-t echo"
    },
    @{
        ID = 6
        Name = "i9-13ks"
        SSHHost = "i9-13ks"
        SSHPort = 22146
        SSHArgs = "-t -p 22146 i9-13ks"
    }
)

$script:ControllerScriptBase64 = $null
$script:AttachHelperScriptBase64 = $null
$script:WorkerBasePathCache = @{}

function Build-SshArgsFromParts {
    param(
        [pscustomobject]$Parts,
        [switch]$Interactive
    )

    $args = @('-o','ServerAliveInterval=60','-o','ServerAliveCountMax=30')
    if ($Interactive -and $Parts.RequestPty) {
        $args += '-t'
    }
    elseif (-not $Interactive) {
        $args += '-T'
    }

    $args += $Parts.Options

    if ($Parts.Port) {
        $args += '-p'
        $args += $Parts.Port
    }

    $args += $Parts.Host
    return $args
}

function Build-ScpArgsFromParts {
    param(
        [pscustomobject]$Parts
    )

    $args = @('-o','ServerAliveInterval=60','-o','ServerAliveCountMax=30')
    $args += $Parts.Options
    if ($Parts.Port) {
        $args += '-P'
        $args += $Parts.Port
    }
    return $args
}

function Get-SshArgs {
    param(
        [object]$Worker,
        [switch]$Interactive
    )

    $parts = Get-WorkerConnectionParts -RawArgs $Worker.SSHArgs -DefaultHost $Worker.Name
    return Build-SshArgsFromParts -Parts $parts -Interactive:$Interactive
}

function Get-WorkerBasePath {
    param(
        [object]$Worker,
        [pscustomobject]$ConnectionParts = $null
    )

    if ($script:WorkerBasePathCache.ContainsKey($Worker.Name)) {
        return $script:WorkerBasePathCache[$Worker.Name]
    }

    if (-not $ConnectionParts) {
        $ConnectionParts = Get-WorkerConnectionParts -RawArgs $Worker.SSHArgs -DefaultHost $Worker.Name
    }

    $sshArgs = Build-SshArgsFromParts -Parts $ConnectionParts -Interactive:$false
    $scriptBlock = "`$ProgressPreference='SilentlyContinue'; [Environment]::GetFolderPath('LocalApplicationData')"
    $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($scriptBlock))
    $remoteCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand $encoded"
    $output = & ssh @sshArgs $remoteCmd
    if ($LASTEXITCODE -ne 0) {
        throw "Unable to determine LocalAppData on $($Worker.Name)."
    }

    $base = ($output | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Last 1).Trim()
    if (-not $base) {
        throw "Unable to read LocalAppData path on $($Worker.Name)."
    }

    $final = Join-Path $base 'UnifiedWorkers'
    $script:WorkerBasePathCache[$Worker.Name] = $final
    return $final
}

function Get-FileBase64 {
    param([string]$Path)
    [Convert]::ToBase64String([IO.File]::ReadAllBytes($Path))
}

function Get-ControllerScriptBase64 {
    if (-not $script:ControllerScriptBase64) {
        $controllerPath = Join-Path $PSScriptRoot 'remote_worker_controller.ps1'
        $script:ControllerScriptBase64 = Get-FileBase64 -Path $controllerPath
    }
    return $script:ControllerScriptBase64
}

function Get-AttachHelperScriptBase64 {
    if (-not $script:AttachHelperScriptBase64) {
        $helperPath = Join-Path $PSScriptRoot 'remote_worker_attach.ps1'
        $script:AttachHelperScriptBase64 = Get-FileBase64 -Path $helperPath
    }
    return $script:AttachHelperScriptBase64
}

function ConvertTo-Base64Unicode {
    param([string]$Content)
    [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($Content))
}

function Get-WorkerSshArgs {
    param([string]$RawArgs)
    if ([string]::IsNullOrWhiteSpace($RawArgs)) {
        return @()
    }
    return $RawArgs -split '\s+' | Where-Object { $_.Trim().Length -gt 0 }
}

function Get-WorkerConnectionParts {
    param(
        [string]$RawArgs,
        [string]$DefaultHost
    )

    $tokens = Get-WorkerSshArgs -RawArgs $RawArgs
    $options = New-Object System.Collections.Generic.List[string]
    $targetHost = $null
    $port = $null
    $requestPty = $false
    $optionsWithArgs = @('-i','-o','-c','-D','-E','-F','-I','-J','-L','-l','-m','-O','-Q','-R','-S','-W','-w')

    for ($i = 0; $i -lt $tokens.Count; $i++) {
        $token = $tokens[$i]
        if ($token -eq '-t' -or $token -eq '-tt') {
            $requestPty = $true
            continue
        }

        if ($token -eq '-p' -and ($i + 1) -lt $tokens.Count) {
            $port = $tokens[$i + 1]
            $i++
            continue
        }

        if ($token.StartsWith('-')) {
            $options.Add($token)
            if ($optionsWithArgs -contains $token -and ($i + 1) -lt $tokens.Count) {
                $options.Add($tokens[$i + 1])
                $i++
            }
            continue
        }

        if (-not $targetHost) {
            $targetHost = $token
            continue
        }

        $options.Add($token)
    }

    if (-not $targetHost) {
        $targetHost = $DefaultHost
    }

    return [pscustomobject]@{
        Host       = $targetHost
        Options    = $options.ToArray()
        Port       = $port
        RequestPty = $requestPty
    }
}

function Invoke-RemotePowerShell {
    param(
        [Parameter(Mandatory = $true)][object]$Worker,
        [Parameter(Mandatory = $true)][string]$Script,
        [switch]$Interactive
    )

    $parts = Get-WorkerConnectionParts -RawArgs $Worker.SSHArgs -DefaultHost $Worker.Name
    if (-not $parts.Host) {
        throw "Unable to determine SSH host for $($Worker.Name)"
    }

    $sshBaseArgs = Build-SshArgsFromParts -Parts $parts -Interactive:$Interactive
    $remoteBasePath = Get-WorkerBasePath -Worker $Worker -ConnectionParts $parts
    $localTemp = [System.IO.Path]::GetTempFileName() + '.ps1'
    Set-Content -Path $localTemp -Value $Script -Encoding UTF8

    $remoteTmpDir = Join-Path $remoteBasePath 'tmp'
    $remoteScriptWin = Join-Path $remoteTmpDir ("script-{0}.ps1" -f ([guid]::NewGuid().ToString()))
    $remoteScriptScp = $remoteScriptWin -replace '\\','/'
    $remoteTarget = "{0}:{1}" -f $parts.Host, ('"'+$remoteScriptScp+'"')

    $ensureScript = "New-Item -ItemType Directory -Path '$remoteTmpDir' -Force | Out-Null"
    $ensureCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($ensureScript))
    & ssh @sshBaseArgs $ensureCmd
    if ($LASTEXITCODE -ne 0) {
        Remove-Item $localTemp -ErrorAction SilentlyContinue
        return $LASTEXITCODE
    }

    $scpArgs = Build-ScpArgsFromParts -Parts $parts
    $scpArgs += $localTemp
    $scpArgs += $remoteTarget

    & scp @scpArgs
    $scpExit = $LASTEXITCODE
    Remove-Item $localTemp -ErrorAction SilentlyContinue
    if ($scpExit -ne 0) {
        return $scpExit
    }

    $execCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$remoteScriptWin`""
    & ssh @sshBaseArgs $execCmd
    $execExit = $LASTEXITCODE

    $cleanupScript = "Remove-Item -LiteralPath '$remoteScriptWin' -ErrorAction SilentlyContinue"
    $cleanupCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -EncodedCommand " + [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($cleanupScript))
    & ssh @sshBaseArgs $cleanupCmd | Out-Null

    return $execExit
}

function Ensure-ControllerDeployed {
    param([object]$Worker)
    $controllerBase64 = Get-ControllerScriptBase64
    $script = @"
`$ProgressPreference = 'SilentlyContinue'
`$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$controllerPath = Join-Path `$dataRoot 'controller.ps1'
[IO.File]::WriteAllBytes(`$controllerPath, [Convert]::FromBase64String('$controllerBase64'))
"@
    Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
}

function Ensure-AttachHelperDeployed {
    param([object]$Worker)
    $helperBase64 = Get-AttachHelperScriptBase64
    $script = @"
`$ProgressPreference = 'SilentlyContinue'
`$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
New-Item -ItemType Directory -Path `$dataRoot -Force | Out-Null
`$attachPath = Join-Path `$dataRoot 'attach-helper.ps1'
[IO.File]::WriteAllBytes(`$attachPath, [Convert]::FromBase64String('$helperBase64'))
"@
    Invoke-RemotePowerShell -Worker $Worker -Script $script | Out-Null
}

function Get-EnsureWorkerScript {
    param(
        [string]$WorkerName,
        [string]$WorkerType,
        [string]$PayloadBase64
    )

    $payload = @{
        WorkerName    = $WorkerName
        WorkerType    = $WorkerType
        PayloadBase64 = $PayloadBase64
    } | ConvertTo-Json -Compress

    $payloadBase64 = ConvertTo-Base64Unicode -Content $payload

    return @"
`$ProgressPreference = 'SilentlyContinue'
`$params = ConvertFrom-Json ([Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('$payloadBase64')))
`$workerName = `$params.WorkerName
`$workerType = `$params.WorkerType
`$payloadBase64 = `$params.PayloadBase64
`$dataRoot = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'UnifiedWorkers'
`$instanceRoot = Join-Path (Join-Path `$dataRoot `$workerType) `$workerName
`$metaPath = Join-Path `$instanceRoot 'state\worker-info.json'
`$controllerPath = Join-Path `$dataRoot 'controller.ps1'

if (-not (Test-Path `$controllerPath)) {
    throw "Controller missing at `$controllerPath"
}

`$shouldStart = `$true
if (Test-Path `$metaPath) {
    try {
        `$meta = Get-Content `$metaPath -Raw | ConvertFrom-Json
        if (`$meta.Status -eq 'running' -and `$meta.WorkerPid) {
            if (Get-Process -Id `$meta.WorkerPid -ErrorAction SilentlyContinue) {
                Write-Host "Worker `$workerName already running (PID `$($meta.WorkerPid))."
                `$shouldStart = `$false
            }
        }
    } catch {
        Write-Host "Unable to parse metadata for `$workerName. Starting new controller." -ForegroundColor Yellow
    }
}

if (`$shouldStart) {
    `$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue
    if (`$pwsh) {
        `$psExe = `$pwsh.Source
    }
    else {
        `$psExe = (Get-Command powershell -ErrorAction Stop).Source
    }

    `$controllerArgs = @(
        '-NoLogo','-NoProfile','-ExecutionPolicy','Bypass',
        '-File',"`$controllerPath",
        '-WorkerName',"`$workerName",
        '-WorkerType',"`$workerType",
        '-PayloadBase64',"`$payloadBase64"
    )

    Start-Process -FilePath `$psExe -ArgumentList `$controllerArgs -WindowStyle Hidden | Out-Null
    Write-Host "Worker `$workerName started under controller." -ForegroundColor Green
}
"@
}

function Ensure-PersistentWorker {
    param(
        [object]$Worker,
        [string]$WorkerType,
        [string]$PayloadScript
    )

    Write-Host "[$($Worker.Name)] Ensuring worker is running under controller..." -ForegroundColor Cyan
    Ensure-ControllerDeployed -Worker $Worker
    $payloadBase64 = ConvertTo-Base64Unicode -Content $PayloadScript
    $ensureScript = Get-EnsureWorkerScript -WorkerName $Worker.Name -WorkerType $WorkerType -PayloadBase64 $payloadBase64
    $result = Invoke-RemotePowerShell -Worker $Worker -Script $ensureScript
    if ($result -ne 0) {
        Write-Host "[$($Worker.Name)] Remote ensure command exited with code $result." -ForegroundColor Yellow
    }
}

function Invoke-WorkerAttach {
    param(
        [object]$Worker,
        [string]$WorkerType,
        [switch]$CommandOnly,
        [string]$Command
    )

    Ensure-AttachHelperDeployed -Worker $Worker
    $paramsBlock = @("-WorkerName","$($Worker.Name)","-WorkerType","$WorkerType")
    if ($CommandOnly) {
        $paramsBlock += "-CommandOnly"
    }
    if ($Command) {
        $paramsBlock += "-Command"
        $paramsBlock += $Command
    }

    $parts = Get-WorkerConnectionParts -RawArgs $Worker.SSHArgs -DefaultHost $Worker.Name
    $sshArgs = Build-SshArgsFromParts -Parts $parts -Interactive:(!$CommandOnly)
    $remoteBasePath = Get-WorkerBasePath -Worker $Worker -ConnectionParts $parts
    $remoteHelper = Join-Path $remoteBasePath 'attach-helper.ps1'
    $quotedArgs = ($paramsBlock | ForEach-Object { '"' + ($_ -replace '"','""') + '"' }) -join ' '
    $remoteCmd = "powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$remoteHelper`" $quotedArgs"

    & ssh @sshArgs $remoteCmd
}

# FUNCTIONS

# This function generates the standard PowerShell remote command
function Get-RemoteStandardWorkerCommand {
    @'
Write-Host "Setting up network connections..." -ForegroundColor Cyan

# Define arrays of drives and network paths
$drives = @('A:', 'F:', 'N:', 'P:')
$networkPaths = @(
    '\\NEXUS\amazon',
    '\\NEXUS\flamenco',
    '\\NEXUS\proj',
    '\\NAS\amazon'
)

# Disconnect all existing connections
Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow
foreach ($path in $networkPaths) { net use $path /delete /y 2>$null }
foreach ($drive in $drives) { net use $drive /delete /y 2>$null }
Write-Host "All network connections cleared." -ForegroundColor Green

# Check if any workers are running
$workerProcesses = Get-Process -Name "flamenco-worker" -ErrorAction SilentlyContinue
if ($workerProcesses) {
    Write-Host "Found $(($workerProcesses | Measure-Object).Count) running Flamenco workers." -ForegroundColor Yellow
    Write-Host "Running workers will NOT be stopped." -ForegroundColor Yellow
} else {
    Write-Host "No running Flamenco workers found." -ForegroundColor Green
}

# Connect to network shares
Write-Host "Establishing network connections..." -ForegroundColor Cyan

# Connect to NEXUS with password automatically supplied
net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes
if ($LASTEXITCODE -eq 0) {
    # Map all NEXUS drives
    net use A: \\NEXUS\amazon /persistent:yes
    net use F: \\NEXUS\flamenco /persistent:yes
    net use P: \\NEXUS\proj /persistent:yes
} else {
    Write-Host "Failed to connect to NEXUS" -ForegroundColor Red
    exit 1  # Exit with error code to trigger restart
}

# Connect to NAS with password automatically supplied
net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes
if ($LASTEXITCODE -ne 0) {
    Write-Host "Failed to connect to NAS" -ForegroundColor Red
    exit 1  # Exit with error code to trigger restart
}

# Verify connections
Write-Host "Current network connections:" -ForegroundColor Cyan
net use

# Start worker
Write-Host "Starting Flamenco worker..." -ForegroundColor Cyan
Set-Location 'F:\software\Flamenco 3.7'
if (Test-Path 'flamenco-worker.exe') {
    Write-Host "Running flamenco-worker.exe..." -ForegroundColor Green
    # Run the worker and capture its exit code
    $workerProcess = Start-Process -FilePath '.\flamenco-worker.exe' -NoNewWindow -PassThru -Wait
    $exitCode = $workerProcess.ExitCode
    Write-Host "Flamenco worker process has terminated with exit code: $exitCode" -ForegroundColor Yellow
    exit $exitCode  # Exit with the worker's exit code to trigger restart if needed
} else {
    Write-Host "Error: flamenco-worker.exe not found in F:\software\Flamenco 3.7" -ForegroundColor Red
    exit 1  # Exit with error code to trigger restart
}
'@
}

# This function generates the CMD PowerShell remote command
function Get-RemoteCmdWorkerCommand {
    @'
Write-Host "Setting up network connections..." -ForegroundColor Cyan

# Define arrays of drives and network paths
$drives = @('A:', 'F:', 'N:', 'P:')
$networkPaths = @(
    '\\NEXUS\amazon',
    '\\NEXUS\flamenco',
    '\\NEXUS\proj',
    '\\NAS\amazon'
)

# Disconnect all existing connections
Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow
foreach ($path in $networkPaths) { net use $path /delete /y 2>$null }
foreach ($drive in $drives) { net use $drive /delete /y 2>$null }
Write-Host "All network connections cleared." -ForegroundColor Green

# Connect to network shares
Write-Host "Establishing network connections..." -ForegroundColor Cyan

# Connect to NEXUS with password automatically supplied
net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes
if ($LASTEXITCODE -eq 0) {
    # Map all NEXUS drives
    net use A: \\NEXUS\amazon /persistent:yes
    net use F: \\NEXUS\flamenco /persistent:yes
    net use P: \\NEXUS\proj /persistent:yes
} else {
    Write-Host "Failed to connect to NEXUS" -ForegroundColor Red
}

# Connect to NAS with password automatically supplied
net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes

# Verify connections
Write-Host "Current network connections:" -ForegroundColor Cyan
net use

# Start worker via CMD - hardcoded paths 
Write-Host "Running command file..." -ForegroundColor Cyan
$defaultCmdPath = "F:\software\Flamenco 3.7\run-flamenco-worker.cmd"

if (Test-Path $defaultCmdPath) {
    Set-Location "F:\software\Flamenco 3.7"
    Write-Host "Starting worker..." -ForegroundColor Green
    # Use hardcoded path to avoid variable expansion issues
    cmd.exe /c "F:\software\Flamenco 3.7\run-flamenco-worker.cmd"
    Write-Host "Worker process has terminated." -ForegroundColor Yellow
} else {
    Write-Host "Command file not found at default location." -ForegroundColor Red
    $customPath = Read-Host "Enter path to .cmd file"
    
    if (Test-Path $customPath) {
        $customDir = Split-Path -Parent $customPath
        Set-Location $customDir
        Write-Host "Starting worker from custom path..." -ForegroundColor Green
        # For custom path, we need to use the variable but in a different way
        Invoke-Expression "cmd.exe /c `"$customPath`""
        Write-Host "Worker process has terminated." -ForegroundColor Yellow
    } else {
        Write-Host "Custom path not found." -ForegroundColor Red
    }
}
'@
}

# This function generates a simplified CMD worker command specifically for Launch All functionality
function Get-RemoteSimplifiedCmdWorkerCommand {
    @'
Write-Host "Setting up network connections..." -ForegroundColor Cyan

# Define arrays of drives and network paths
$drives = @('A:', 'F:', 'N:', 'P:')
$networkPaths = @(
    '\\NEXUS\amazon',
    '\\NEXUS\flamenco',
    '\\NEXUS\proj',
    '\\NAS\amazon'
)

# Disconnect all existing connections
Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow
foreach ($path in $networkPaths) { net use $path /delete /y 2>$null }
foreach ($drive in $drives) { net use $drive /delete /y 2>$null }
Write-Host "All network connections cleared." -ForegroundColor Green

# Connect to network shares
Write-Host "Establishing network connections..." -ForegroundColor Cyan

# Connect to NEXUS with password automatically supplied
net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes
if ($LASTEXITCODE -eq 0) {
    # Map all NEXUS drives
    net use A: \\NEXUS\amazon /persistent:yes
    net use F: \\NEXUS\flamenco /persistent:yes
    net use P: \\NEXUS\proj /persistent:yes
} else {
    Write-Host "Failed to connect to NEXUS" -ForegroundColor Red
}

# Connect to NAS with password automatically supplied
net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes

# Verify connections
Write-Host "Current network connections:" -ForegroundColor Cyan
net use

# Simple direct command execution with automatic "2" input
Write-Host "Running Flamenco worker..." -ForegroundColor Cyan
Set-Location -Path "F:\software\Flamenco 3.7"
if (Test-Path -Path "run-flamenco-worker.cmd") {
    # Create a temporary file to store the "2" input
    $tempInputFile = [System.IO.Path]::GetTempFileName()
    Set-Content -Path $tempInputFile -Value "2"
    # Run the command with input redirected from our temp file
    cmd.exe /c "run-flamenco-worker.cmd < $tempInputFile"
    # Clean up the temp file
    Remove-Item -Path $tempInputFile -Force
    Write-Host "Worker process has terminated." -ForegroundColor Yellow
} else {
    Write-Host "Worker command file not found." -ForegroundColor Red
}
'@
}

# This function launches the standard worker
function Start-StandardWorker {
    param (
        [Parameter(Mandatory = $true)]
        [object]$Worker
    )

    $payloadScript = Get-RemoteStandardWorkerCommand
    Ensure-PersistentWorker -Worker $Worker -WorkerType 'flamenco' -PayloadScript $payloadScript
    Invoke-WorkerAttach -Worker $Worker -WorkerType 'flamenco'
}

# This function launches the CMD worker
function Start-CmdWorker {
    param (
        [Parameter(Mandatory = $true)]
        [object]$Worker
    )

    $payloadScript = Get-RemoteCmdWorkerCommand
    Ensure-PersistentWorker -Worker $Worker -WorkerType 'flamenco' -PayloadScript $payloadScript
    Invoke-WorkerAttach -Worker $Worker -WorkerType 'flamenco'
}

# This function launches ALL workers in Windows Terminal tabs
function Start-AllWorkers {
    param (
        [Parameter(Mandatory = $true)]
        [string]$WorkerType
    )

    Write-Host "Ensuring ALL $WorkerType workers are running under controllers..." -ForegroundColor Cyan

    $payloadScript = if ($WorkerType -eq 'CMD') {
        Get-RemoteCmdWorkerCommand
    } else {
        Get-RemoteStandardWorkerCommand
    }

    foreach ($worker in $workers) {
        Ensure-PersistentWorker -Worker $worker -WorkerType 'flamenco' -PayloadScript $payloadScript
    }

    Write-Host "All workers processed. Attach to any worker from the menu to monitor output." -ForegroundColor Green
    Write-Host "Press Enter to return to the menu..." -ForegroundColor Green
    Read-Host | Out-Null
}

# Main menu loop
$exitRequested = $false
while (-not $exitRequested) {
    Clear-Host
    # Display main menu
    Write-Host "==========================================" -ForegroundColor Cyan
    Write-Host "    UNIFIED FLAMENCO WORKER LAUNCHER" -ForegroundColor Cyan
    Write-Host "==========================================" -ForegroundColor Cyan
    Write-Host
    Write-Host "Main Menu:" -ForegroundColor Magenta
    Write-Host "1. Launch Standard Worker (direct worker execution)" -ForegroundColor Yellow
    Write-Host "2. Launch CMD Worker (uses .cmd file - more resilient)" -ForegroundColor Yellow
    Write-Host "3. Exit" -ForegroundColor Yellow
    Write-Host
    
    $mainSelection = Read-Host "Select option (1-3)"
    
    switch ($mainSelection) {
        "1" {
            # Standard Worker Menu
            Clear-Host
            Write-Host "==========================================" -ForegroundColor Cyan
            Write-Host "    STANDARD WORKER SELECTION" -ForegroundColor Cyan
            Write-Host "==========================================" -ForegroundColor Cyan
            Write-Host
            Write-Host "Select a system to connect to:" -ForegroundColor Yellow
            
            foreach ($worker in $workers) {
                Write-Host "$($worker.ID). $($worker.Name)" -ForegroundColor Green
            }
            
            Write-Host "0. Launch ALL workers (separate windows)" -ForegroundColor Magenta
            Write-Host "B. Back to main menu" -ForegroundColor Yellow
            Write-Host
            
            $workerSelection = Read-Host "Enter your selection"
            
            if ($workerSelection -eq "B" -or $workerSelection -eq "b") {
                # Go back to main menu
                continue
            }
            elseif ($workerSelection -eq "0") {
                # Launch all standard workers - handle this specifically to avoid confusion with worker IDs
                Write-Host "Selected: Launch ALL workers" -ForegroundColor Cyan
                Start-AllWorkers -WorkerType "Standard"
            }
            else {
                # Try to convert to integer and launch selected worker
                try {
                    $selectedID = [int]$workerSelection
                    $selectedWorker = $workers | Where-Object { $_.ID -eq $selectedID }
                    
                    if ($selectedWorker) {
                        Start-StandardWorker -Worker $selectedWorker
                    }
                    else {
                        Write-Host "Invalid selection. Worker ID $selectedID not found." -ForegroundColor Red
                        Start-Sleep -Seconds 2
                    }
                }
                catch {
                    Write-Host "Invalid selection. Please try again." -ForegroundColor Red
                    Start-Sleep -Seconds 2
                }
            }
        }
        "2" {
            # CMD Worker Menu
            Clear-Host
            Write-Host "==========================================" -ForegroundColor Cyan
            Write-Host "    CMD WORKER SELECTION" -ForegroundColor Cyan
            Write-Host "==========================================" -ForegroundColor Cyan
            Write-Host
            Write-Host "Select a system to connect to:" -ForegroundColor Yellow
            
            foreach ($worker in $workers) {
                Write-Host "$($worker.ID). $($worker.Name)" -ForegroundColor Green
            }
            
            Write-Host "0. Launch ALL workers (separate windows)" -ForegroundColor Magenta
            Write-Host "B. Back to main menu" -ForegroundColor Yellow
            Write-Host
            
            $cmdSelection = Read-Host "Enter your selection"
            
            if ($cmdSelection -eq "B" -or $cmdSelection -eq "b") {
                # Go back to main menu
                continue
            }
            elseif ($cmdSelection -eq "0") {
                # Launch all CMD workers - handle this specifically to avoid confusion with worker IDs
                Write-Host "Selected: Launch ALL workers" -ForegroundColor Cyan
                Start-AllWorkers -WorkerType "CMD"
            }
            else {
                # Try to convert to integer and launch selected worker
                try {
                    $selectedID = [int]$cmdSelection
                    $selectedWorker = $workers | Where-Object { $_.ID -eq $selectedID }
                    
                    if ($selectedWorker) {
                        Start-CmdWorker -Worker $selectedWorker
                    }
                    else {
                        Write-Host "Invalid selection. Worker ID $selectedID not found." -ForegroundColor Red
                        Start-Sleep -Seconds 2
                    }
                }
                catch {
                    Write-Host "Invalid selection. Please try again." -ForegroundColor Red
                    Start-Sleep -Seconds 2
                }
            }
        }
        "3" {
            # Exit
            $exitRequested = $true
        }
        default {
            Write-Host "Invalid selection. Please try again." -ForegroundColor Red
            Start-Sleep -Seconds 2
        }
    }
}

Write-Host "`nExiting Unified Flamenco Launcher. Goodbye!" -ForegroundColor Cyan