# Master Unified Flamenco Launcher Script Write-Host "==========================================" -ForegroundColor Cyan Write-Host " UNIFIED FLAMENCO WORKER LAUNCHER" -ForegroundColor Cyan Write-Host "==========================================" -ForegroundColor Cyan Write-Host # Define worker-specific configuration $workers = @( @{ ID = 1 Name = "i9kf" SSHHost = "i9kf" SSHPort = 22 SSHArgs = "-t i9kf" }, @{ ID = 2 Name = "blender-boss" SSHHost = "blender-boss" SSHPort = 22 SSHArgs = "-t blender-boss" }, @{ ID = 3 Name = "max" SSHHost = "max" SSHPort = 22 SSHArgs = "-t max" }, @{ ID = 4 Name = "masterbox" SSHHost = "masterbox" SSHPort = 22 SSHArgs = "-t masterbox" }, @{ ID = 5 Name = "echo" SSHHost = "echo" SSHPort = 22 SSHArgs = "-t echo" }, @{ ID = 6 Name = "i9-13ks" SSHHost = "i9-13ks" SSHPort = 22146 SSHArgs = "-t -p 22146 i9-13ks" } ) # FUNCTIONS # This function generates the standard PowerShell remote command function Get-RemoteStandardWorkerCommand { @' Write-Host "Setting up network connections..." -ForegroundColor Cyan # Define arrays of drives and network paths $drives = @('A:', 'F:', 'N:', 'P:') $networkPaths = @( '\\NEXUS\amazon', '\\NEXUS\flamenco', '\\NEXUS\proj', '\\NAS\amazon' ) # Disconnect all existing connections Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow foreach ($path in $networkPaths) { net use $path /delete /y 2>$null } foreach ($drive in $drives) { net use $drive /delete /y 2>$null } Write-Host "All network connections cleared." -ForegroundColor Green # Check if any workers are running $workerProcesses = Get-Process -Name "flamenco-worker" -ErrorAction SilentlyContinue if ($workerProcesses) { Write-Host "Found $(($workerProcesses | Measure-Object).Count) running Flamenco workers." -ForegroundColor Yellow Write-Host "Running workers will NOT be stopped." -ForegroundColor Yellow } else { Write-Host "No running Flamenco workers found." -ForegroundColor Green } # Connect to network shares Write-Host "Establishing network connections..." -ForegroundColor Cyan # Connect to NEXUS with password automatically supplied net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes if ($LASTEXITCODE -eq 0) { # Map all NEXUS drives net use A: \\NEXUS\amazon /persistent:yes net use F: \\NEXUS\flamenco /persistent:yes net use P: \\NEXUS\proj /persistent:yes } else { Write-Host "Failed to connect to NEXUS" -ForegroundColor Red exit 1 # Exit with error code to trigger restart } # Connect to NAS with password automatically supplied net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes if ($LASTEXITCODE -ne 0) { Write-Host "Failed to connect to NAS" -ForegroundColor Red exit 1 # Exit with error code to trigger restart } # Verify connections Write-Host "Current network connections:" -ForegroundColor Cyan net use # Start worker Write-Host "Starting Flamenco worker..." -ForegroundColor Cyan if (Test-Path 'C:\Program Files\Blender Foundation\Flamenco 3.6') { Set-Location 'C:\Program Files\Blender Foundation\Flamenco 3.6' if (Test-Path 'flamenco-worker.exe') { Write-Host "Running flamenco-worker.exe..." -ForegroundColor Green # Run the worker and capture its exit code $workerProcess = Start-Process -FilePath '.\flamenco-worker.exe' -NoNewWindow -PassThru -Wait $exitCode = $workerProcess.ExitCode Write-Host "Flamenco worker process has terminated with exit code: $exitCode" -ForegroundColor Yellow exit $exitCode # Exit with the worker's exit code to trigger restart if needed } else { Write-Host "Error: flamenco-worker.exe not found" -ForegroundColor Red exit 1 # Exit with error code to trigger restart } } else { Write-Host "Error: Flamenco directory not found" -ForegroundColor Red exit 1 # Exit with error code to trigger restart } '@ } # This function generates the CMD PowerShell remote command function Get-RemoteCmdWorkerCommand { @' Write-Host "Setting up network connections..." -ForegroundColor Cyan # Define arrays of drives and network paths $drives = @('A:', 'F:', 'N:', 'P:') $networkPaths = @( '\\NEXUS\amazon', '\\NEXUS\flamenco', '\\NEXUS\proj', '\\NAS\amazon' ) # Disconnect all existing connections Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow foreach ($path in $networkPaths) { net use $path /delete /y 2>$null } foreach ($drive in $drives) { net use $drive /delete /y 2>$null } Write-Host "All network connections cleared." -ForegroundColor Green # Connect to network shares Write-Host "Establishing network connections..." -ForegroundColor Cyan # Connect to NEXUS with password automatically supplied net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes if ($LASTEXITCODE -eq 0) { # Map all NEXUS drives net use A: \\NEXUS\amazon /persistent:yes net use F: \\NEXUS\flamenco /persistent:yes net use P: \\NEXUS\proj /persistent:yes } else { Write-Host "Failed to connect to NEXUS" -ForegroundColor Red } # Connect to NAS with password automatically supplied net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes # Verify connections Write-Host "Current network connections:" -ForegroundColor Cyan net use # Start worker via CMD - hardcoded paths Write-Host "Running command file..." -ForegroundColor Cyan $defaultCmdPath = "F:\software\Flamenco 3.6\run-flamenco-worker.cmd" if (Test-Path $defaultCmdPath) { Set-Location "F:\software\Flamenco 3.6" Write-Host "Starting worker..." -ForegroundColor Green # Use hardcoded path to avoid variable expansion issues cmd.exe /c "F:\software\Flamenco 3.6\run-flamenco-worker.cmd" Write-Host "Worker process has terminated." -ForegroundColor Yellow } else { Write-Host "Command file not found at default location." -ForegroundColor Red $customPath = Read-Host "Enter path to .cmd file" if (Test-Path $customPath) { $customDir = Split-Path -Parent $customPath Set-Location $customDir Write-Host "Starting worker from custom path..." -ForegroundColor Green # For custom path, we need to use the variable but in a different way Invoke-Expression "cmd.exe /c `"$customPath`"" Write-Host "Worker process has terminated." -ForegroundColor Yellow } else { Write-Host "Custom path not found." -ForegroundColor Red } } '@ } # This function generates a simplified CMD worker command specifically for Launch All functionality function Get-RemoteSimplifiedCmdWorkerCommand { @' Write-Host "Setting up network connections..." -ForegroundColor Cyan # Define arrays of drives and network paths $drives = @('A:', 'F:', 'N:', 'P:') $networkPaths = @( '\\NEXUS\amazon', '\\NEXUS\flamenco', '\\NEXUS\proj', '\\NAS\amazon' ) # Disconnect all existing connections Write-Host "Disconnecting existing network connections..." -ForegroundColor Yellow foreach ($path in $networkPaths) { net use $path /delete /y 2>$null } foreach ($drive in $drives) { net use $drive /delete /y 2>$null } Write-Host "All network connections cleared." -ForegroundColor Green # Connect to network shares Write-Host "Establishing network connections..." -ForegroundColor Cyan # Connect to NEXUS with password automatically supplied net use \\NEXUS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes if ($LASTEXITCODE -eq 0) { # Map all NEXUS drives net use A: \\NEXUS\amazon /persistent:yes net use F: \\NEXUS\flamenco /persistent:yes net use P: \\NEXUS\proj /persistent:yes } else { Write-Host "Failed to connect to NEXUS" -ForegroundColor Red } # Connect to NAS with password automatically supplied net use N: \\NAS\amazon /user:Nathan HeadsTalk1ng! /persistent:yes # Verify connections Write-Host "Current network connections:" -ForegroundColor Cyan net use # Simple direct command execution with automatic "2" input Write-Host "Running Flamenco worker..." -ForegroundColor Cyan Set-Location -Path "F:\software\Flamenco` 3.6" if (Test-Path -Path "run-flamenco-worker.cmd") { # Create a temporary file to store the "2" input $tempInputFile = [System.IO.Path]::GetTempFileName() Set-Content -Path $tempInputFile -Value "2" # Run the command with input redirected from our temp file cmd.exe /c "run-flamenco-worker.cmd < $tempInputFile" # Clean up the temp file Remove-Item -Path $tempInputFile -Force Write-Host "Worker process has terminated." -ForegroundColor Yellow } else { Write-Host "Worker command file not found." -ForegroundColor Red } '@ } # This function launches the standard worker function Start-StandardWorker { param ( [Parameter(Mandatory = $true)] [object]$Worker ) $retryCount = 0 $retryDelay = 15 # seconds between retries $workerRestarted = $false while ($true) { # Changed to infinite loop if ($retryCount -gt 0) { Write-Host "`nRestarting worker process (Attempt $($retryCount + 1))..." -ForegroundColor Yellow Start-Sleep -Seconds $retryDelay $workerRestarted = $true } Write-Host "Connecting to $($Worker.Name)..." -ForegroundColor Cyan if ($workerRestarted) { Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow } try { $remoteCommand = Get-RemoteStandardWorkerCommand # Encode the command to handle special characters $bytes = [System.Text.Encoding]::Unicode.GetBytes($remoteCommand) $encodedCommand = [Convert]::ToBase64String($bytes) # Execute the encoded command on the remote machine Write-Host "Connecting to $($Worker.Name) and executing worker script..." -ForegroundColor Yellow # Add SSH keepalive settings to reduce chance of random disconnections $sshCommand = "ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($Worker.SSHArgs) ""powershell -EncodedCommand $encodedCommand""" # Execute the SSH command and capture the exit code Invoke-Expression $sshCommand $sshExitCode = $LASTEXITCODE # Check if SSH command completed successfully if ($sshExitCode -eq 0) { Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green Start-Sleep -Seconds 2 # Brief pause before restarting $retryCount = 0 # Reset counter for successful completion continue # Continue the loop instead of breaking } else { throw "Worker process exited with code: $sshExitCode" } } catch { $retryCount++ Write-Host "`nAn error occurred while running worker on $($Worker.Name):" -ForegroundColor Red Write-Host $_.Exception.Message -ForegroundColor Red Write-Host "`nAttempting to restart worker in $retryDelay seconds..." -ForegroundColor Yellow } } } # This function launches the CMD worker function Start-CmdWorker { param ( [Parameter(Mandatory = $true)] [object]$Worker ) $retryCount = 0 $retryDelay = 5 # seconds between retries $workerRestarted = $false while ($true) { # Changed to infinite loop if ($retryCount -gt 0) { Write-Host "`nRestarting worker process (Attempt $($retryCount + 1))..." -ForegroundColor Yellow Start-Sleep -Seconds $retryDelay $workerRestarted = $true } Write-Host "Connecting to $($Worker.Name) (CMD mode)..." -ForegroundColor Cyan if ($workerRestarted) { Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow } try { $remoteCommand = Get-RemoteCmdWorkerCommand # Encode the command to handle special characters $bytes = [System.Text.Encoding]::Unicode.GetBytes($remoteCommand) $encodedCommand = [Convert]::ToBase64String($bytes) # Execute the encoded command on the remote machine Write-Host "Connecting to $($Worker.Name) and executing CMD worker script..." -ForegroundColor Yellow # Add SSH keepalive settings to reduce chance of random disconnections $sshCommand = "ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($Worker.SSHArgs) ""powershell -EncodedCommand $encodedCommand""" # Execute the SSH command and capture the exit code Invoke-Expression $sshCommand $sshExitCode = $LASTEXITCODE # Check if SSH command completed successfully if ($sshExitCode -eq 0) { Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green Start-Sleep -Seconds 2 # Brief pause before restarting $retryCount = 0 # Reset counter for successful completion continue # Continue the loop instead of breaking } else { throw "Worker process exited with code: $sshExitCode" } } catch { $retryCount++ Write-Host "`nAn error occurred while running worker on $($Worker.Name):" -ForegroundColor Red Write-Host $_.Exception.Message -ForegroundColor Red Write-Host "`nAttempting to restart worker in $retryDelay seconds..." -ForegroundColor Yellow } } } # This function launches ALL workers in Windows Terminal tabs function Start-AllWorkers { param ( [Parameter(Mandatory = $true)] [string]$WorkerType ) Write-Host "Launching ALL $WorkerType workers in Windows Terminal tabs..." -ForegroundColor Cyan try { # First, check if Windows Terminal is available if (-not (Get-Command wt.exe -ErrorAction SilentlyContinue)) { Write-Host "Windows Terminal (wt.exe) not found. Falling back to separate windows." -ForegroundColor Yellow $useTerminal = $false } else { $useTerminal = $true } foreach ($worker in $workers) { # Create a new PowerShell script with a unique name for this worker $tempScriptPath = [System.IO.Path]::GetTempFileName() + ".ps1" # Create different script content based on worker type if ($WorkerType -eq "CMD") { # CMD workers get retry logic at the local level $scriptContent = @" # Wrap everything in a try-catch to prevent script termination try { Write-Host 'Launching $WorkerType worker for $($worker.Name)' -ForegroundColor Cyan `$retryCount = 0 `$retryDelay = 5 # seconds between retries `$workerRestarted = `$false while (`$true) { # Changed to infinite loop try { if (`$retryCount -gt 0) { Write-Host "`nRestarting worker process (Attempt `$(`$retryCount + 1))..." -ForegroundColor Yellow Start-Sleep -Seconds `$retryDelay `$workerRestarted = `$true } Write-Host "Connecting to $($worker.Name) ($WorkerType mode)..." -ForegroundColor Cyan if (`$workerRestarted) { Write-Host "Worker was restarted due to disconnection or crash." -ForegroundColor Yellow } # Get remote command `$remoteCommand = @' $(Get-RemoteSimplifiedCmdWorkerCommand) '@ # Encode the command `$bytes = [System.Text.Encoding]::Unicode.GetBytes(`$remoteCommand) `$encodedCommand = [Convert]::ToBase64String(`$bytes) # Execute SSH command with keepalive settings and capture exit code ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($worker.SSHArgs) "powershell -EncodedCommand `$encodedCommand" `$sshExitCode = `$LASTEXITCODE # Check if SSH command completed successfully if (`$sshExitCode -eq 0) { Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green Start-Sleep -Seconds 2 # Brief pause before restarting `$retryCount = 0 # Reset counter for successful completion continue # Continue the loop instead of breaking } else { throw "SSH command failed with exit code: `$sshExitCode" } } catch { `$retryCount++ Write-Host "An error occurred while connecting to $($worker.Name):" -ForegroundColor Red Write-Host `$_.Exception.Message -ForegroundColor Red Write-Host "Attempting to reconnect in `$retryDelay seconds..." -ForegroundColor Yellow # Don't rethrow - we want to continue the retry loop } } } catch { # This outer catch block is for any unexpected errors that might terminate the script Write-Host "`nCRITICAL ERROR: Script encountered an unexpected error:" -ForegroundColor Red Write-Host `$_.Exception.Message -ForegroundColor Red Write-Host "`nRestarting the entire worker process in 5 seconds..." -ForegroundColor Yellow Start-Sleep -Seconds 5 # Restart the script by calling itself & `$MyInvocation.MyCommand.Path } "@ } else { # Standard workers keep the original retry logic $scriptContent = @" # Wrap everything in a try-catch to prevent script termination try { Write-Host 'Launching $WorkerType worker for $($worker.Name)' -ForegroundColor Cyan `$retryCount = 0 `$retryDelay = 5 # seconds between retries while (`$true) { # Changed to infinite loop try { if (`$retryCount -gt 0) { Write-Host "Retry attempt `$retryCount..." -ForegroundColor Yellow Start-Sleep -Seconds `$retryDelay } # Get remote command `$remoteCommand = @' $(Get-RemoteStandardWorkerCommand) '@ # Encode the command `$bytes = [System.Text.Encoding]::Unicode.GetBytes(`$remoteCommand) `$encodedCommand = [Convert]::ToBase64String(`$bytes) # Execute SSH command with keepalive settings ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=30 $($worker.SSHArgs) "powershell -EncodedCommand `$encodedCommand" `$sshExitCode = `$LASTEXITCODE # Check SSH exit code and handle accordingly if (`$sshExitCode -eq 0) { Write-Host "`nWorker completed successfully. Restarting automatically..." -ForegroundColor Green Start-Sleep -Seconds 2 # Brief pause before restarting `$retryCount = 0 # Reset counter for successful completion continue # Continue the loop instead of breaking } else { throw "SSH command failed with exit code: `$sshExitCode" } } catch { `$retryCount++ Write-Host "An error occurred while connecting to $($worker.Name):" -ForegroundColor Red Write-Host `$_.Exception.Message -ForegroundColor Red Write-Host "Attempting to reconnect in `$retryDelay seconds..." -ForegroundColor Yellow # Don't rethrow - we want to continue the retry loop } } } catch { # This outer catch block is for any unexpected errors that might terminate the script Write-Host "`nCRITICAL ERROR: Script encountered an unexpected error:" -ForegroundColor Red Write-Host `$_.Exception.Message -ForegroundColor Red Write-Host "`nRestarting the entire worker process in 5 seconds..." -ForegroundColor Yellow Start-Sleep -Seconds 5 # Restart the script by calling itself & `$MyInvocation.MyCommand.Path } "@ } # Write the script to file Set-Content -Path $tempScriptPath -Value $scriptContent if ($useTerminal) { # Launch in a new Windows Terminal tab $tabTitle = "$($worker.Name) - $WorkerType Worker" Start-Process wt.exe -ArgumentList "-w 0 new-tab --title `"$tabTitle`" powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$tempScriptPath`"" } else { # Fallback to separate window if Windows Terminal is not available Start-Process powershell -ArgumentList "-NoLogo -NoProfile -ExecutionPolicy Bypass -File `"$tempScriptPath`"" } Write-Host "Started $($worker.Name) ($WorkerType) worker in a new tab." -ForegroundColor Green Start-Sleep -Milliseconds 300 # Small delay between launches } Write-Host "`nAll $WorkerType worker scripts have been launched in Windows Terminal tabs." -ForegroundColor Cyan } catch { Write-Host "Error launching workers: $($_.Exception.Message)" -ForegroundColor Red } Write-Host "Press Enter to return to the menu..." -ForegroundColor Green Read-Host } # Main menu loop $exitRequested = $false while (-not $exitRequested) { Clear-Host # Display main menu Write-Host "==========================================" -ForegroundColor Cyan Write-Host " UNIFIED FLAMENCO WORKER LAUNCHER" -ForegroundColor Cyan Write-Host "==========================================" -ForegroundColor Cyan Write-Host Write-Host "Main Menu:" -ForegroundColor Magenta Write-Host "1. Launch Standard Worker (direct worker execution)" -ForegroundColor Yellow Write-Host "2. Launch CMD Worker (uses .cmd file - more resilient)" -ForegroundColor Yellow Write-Host "3. Exit" -ForegroundColor Yellow Write-Host $mainSelection = Read-Host "Select option (1-3)" switch ($mainSelection) { "1" { # Standard Worker Menu Clear-Host Write-Host "==========================================" -ForegroundColor Cyan Write-Host " STANDARD WORKER SELECTION" -ForegroundColor Cyan Write-Host "==========================================" -ForegroundColor Cyan Write-Host Write-Host "Select a system to connect to:" -ForegroundColor Yellow foreach ($worker in $workers) { Write-Host "$($worker.ID). $($worker.Name)" -ForegroundColor Green } Write-Host "0. Launch ALL workers (separate windows)" -ForegroundColor Magenta Write-Host "B. Back to main menu" -ForegroundColor Yellow Write-Host $workerSelection = Read-Host "Enter your selection" if ($workerSelection -eq "B" -or $workerSelection -eq "b") { # Go back to main menu continue } elseif ($workerSelection -eq "0") { # Launch all standard workers - handle this specifically to avoid confusion with worker IDs Write-Host "Selected: Launch ALL workers" -ForegroundColor Cyan Start-AllWorkers -WorkerType "Standard" } else { # Try to convert to integer and launch selected worker try { $selectedID = [int]$workerSelection $selectedWorker = $workers | Where-Object { $_.ID -eq $selectedID } if ($selectedWorker) { Start-StandardWorker -Worker $selectedWorker } else { Write-Host "Invalid selection. Worker ID $selectedID not found." -ForegroundColor Red Start-Sleep -Seconds 2 } } catch { Write-Host "Invalid selection. Please try again." -ForegroundColor Red Start-Sleep -Seconds 2 } } } "2" { # CMD Worker Menu Clear-Host Write-Host "==========================================" -ForegroundColor Cyan Write-Host " CMD WORKER SELECTION" -ForegroundColor Cyan Write-Host "==========================================" -ForegroundColor Cyan Write-Host Write-Host "Select a system to connect to:" -ForegroundColor Yellow foreach ($worker in $workers) { Write-Host "$($worker.ID). $($worker.Name)" -ForegroundColor Green } Write-Host "0. Launch ALL workers (separate windows)" -ForegroundColor Magenta Write-Host "B. Back to main menu" -ForegroundColor Yellow Write-Host $cmdSelection = Read-Host "Enter your selection" if ($cmdSelection -eq "B" -or $cmdSelection -eq "b") { # Go back to main menu continue } elseif ($cmdSelection -eq "0") { # Launch all CMD workers - handle this specifically to avoid confusion with worker IDs Write-Host "Selected: Launch ALL workers" -ForegroundColor Cyan Start-AllWorkers -WorkerType "CMD" } else { # Try to convert to integer and launch selected worker try { $selectedID = [int]$cmdSelection $selectedWorker = $workers | Where-Object { $_.ID -eq $selectedID } if ($selectedWorker) { Start-CmdWorker -Worker $selectedWorker } else { Write-Host "Invalid selection. Worker ID $selectedID not found." -ForegroundColor Red Start-Sleep -Seconds 2 } } catch { Write-Host "Invalid selection. Please try again." -ForegroundColor Red Start-Sleep -Seconds 2 } } } "3" { # Exit $exitRequested = $true } default { Write-Host "Invalid selection. Please try again." -ForegroundColor Red Start-Sleep -Seconds 2 } } } Write-Host "`nExiting Unified Flamenco Launcher. Goodbye!" -ForegroundColor Cyan