From 11f286e87fa4ba0399aff916cdf8c8712a1a8381 Mon Sep 17 00:00:00 2001 From: Raincloud Date: Wed, 17 Dec 2025 00:42:29 -0700 Subject: [PATCH] ConfigLoader now changes config structDir when ran --- ...t-changes-while-respecting-home-pathing.md | 220 ++++++------- ...update-gitignore-files-in-project-repos.md | 54 +-- ...on-failure-and-memory-allocation-issues.md | 310 +++++++++--------- ...7-34Z-configloader-structdir-path-issue.md | 184 +++++++++++ ConfigLoader.ps1 | 26 ++ config.json | 7 +- 6 files changed, 505 insertions(+), 296 deletions(-) create mode 100644 .specstory/history/2025-12-17_07-34Z-configloader-structdir-path-issue.md diff --git a/.specstory/history/2025-08-21_00-15Z-implement-changes-while-respecting-home-pathing.md b/.specstory/history/2025-08-21_00-15Z-implement-changes-while-respecting-home-pathing.md index 94c5475..c31687b 100644 --- a/.specstory/history/2025-08-21_00-15Z-implement-changes-while-respecting-home-pathing.md +++ b/.specstory/history/2025-08-21_00-15Z-implement-changes-while-respecting-home-pathing.md @@ -360,18 +360,18 @@ I can see the issue! The current `UpdateSequences.ps1` is still using the old `d -
Tool use: **code_edit** • Edit file: UpdateSequences.ps1 +
Tool use: **search_replace** • Edit file: UpdateSequences.ps1 **Chunk 1** Lines added: 2, lines removed: 1 ```diff @@ -93,5 +93,6 @@ - "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) started in '$root' ===" | Add-Content -LiteralPath $logFile + "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) started in '$root' ===" | Add-Content -LiteralPath $logFile -- $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter 'daily_*' | Where-Object { $_.Name -ne '_archive' } +- $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter 'daily_*' | Where-Object { $_.Name -ne '_archive' } + # Scan for folders with YYYY-MM-DD format (e.g., 2023-12-15) instead of daily_* -+ $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter '????-??-??' | Where-Object { $_.Name -ne '_archive' } ++ $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter '????-??-??' | Where-Object { $_.Name -ne '_archive' } $mapDaily = @{} ``` @@ -417,7 +417,7 @@ Absolutely! Let me update `UpgradeSeqBatches.ps1` to use your home pathing inste -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: 2, lines removed: 2 @@ -426,10 +426,10 @@ Lines added: 2, lines removed: 2 @@ -1,6 +1,6 @@ # Copy the current UpdateSequences.bat to all subfolders that have it -- $sourceFile = "A:\1 Amazon_Active_Projects\3 ProjectStructure\UpdateSequences.bat" -- $sourcePs1 = "A:\1 Amazon_Active_Projects\3 ProjectStructure\UpdateSequences.ps1" -+ $sourceFile = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.bat" -+ $sourcePs1 = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.ps1" +- $sourceFile = "A:\1 Amazon_Active_Projects\3 ProjectStructure\UpdateSequences.bat" +- $sourcePs1 = "A:\1 Amazon_Active_Projects\3 ProjectStructure\UpdateSequences.ps1" ++ $sourceFile = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.bat" ++ $sourcePs1 = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.ps1" if (-not (Test-Path $sourceFile)) { ``` @@ -442,7 +442,7 @@ Lines added: 2, lines removed: 2 -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: 1, lines removed: 1 @@ -451,10 +451,10 @@ Lines added: 1, lines removed: 1 @@ -20,5 +20,5 @@ # Find all UpdateSequences.bat files -- $targetFiles = Get-ChildItem -Path "A:\1 Amazon_Active_Projects" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } -+ $targetFiles = Get-ChildItem -Path "R:\Creative\artsy\maya" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } +- $targetFiles = Get-ChildItem -Path "A:\1 Amazon_Active_Projects" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } ++ $targetFiles = Get-ChildItem -Path "R:\Creative\artsy\maya" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } - Write-Host "Found $($targetFiles.Count) target files to update:" -ForegroundColor Yellow + Write-Host "Found $($targetFiles.Count) target files to update:" -ForegroundColor Yellow ```
@@ -528,7 +528,7 @@ Let me remove that part from the script: -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: undefined, lines removed: 4 @@ -536,11 +536,11 @@ Lines added: undefined, lines removed: 4 ```diff @@ -40,8 +40,4 @@ Copy-Item -Path $sourceFile -Destination $targetBat -Force - Write-Host "✓ Updated: $targetBat" -ForegroundColor Green + Write-Host "✓ Updated: $targetBat" -ForegroundColor Green - - # Copy the .ps1 file - Copy-Item -Path $sourcePs1 -Destination $targetPs1 -Force -- Write-Host "✓ Updated: $targetPs1" -ForegroundColor Green +- Write-Host "✓ Updated: $targetPs1" -ForegroundColor Green $updatedCount++ ``` @@ -553,7 +553,7 @@ Lines added: undefined, lines removed: 4 -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: undefined, lines removed: 1 @@ -561,8 +561,8 @@ Lines added: undefined, lines removed: 1 ```diff @@ -34,5 +34,4 @@ $targetDir = $targetFile.Directory.FullName - $targetBat = Join-Path $targetDir "UpdateSequences.bat" -- $targetPs1 = Join-Path $targetDir "UpdateSequences.ps1" + $targetBat = Join-Path $targetDir "UpdateSequences.bat" +- $targetPs1 = Join-Path $targetDir "UpdateSequences.ps1" try { ``` @@ -599,7 +599,7 @@ Absolutely! Since we're no longer copying the PS1 files, we don't need that vari -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: undefined, lines removed: 1 @@ -607,8 +607,8 @@ Lines added: undefined, lines removed: 1 ```diff @@ -2,5 +2,4 @@ - $sourceFile = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.bat" -- $sourcePs1 = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.ps1" + $sourceFile = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.bat" +- $sourcePs1 = "R:\Creative\artsy\maya\0 ProjectStructure\UpdateSequences.ps1" if (-not (Test-Path $sourceFile)) { ``` @@ -621,7 +621,7 @@ Lines added: undefined, lines removed: 1 -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: 1, lines removed: 4 @@ -631,12 +631,12 @@ Lines added: 1, lines removed: 4 } - if (-not (Test-Path $sourcePs1)) { -- Write-Error "Source PowerShell file not found: $sourcePs1" +- Write-Error "Source PowerShell file not found: $sourcePs1" - exit 1 - } + - Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan + Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan ```
@@ -647,17 +647,17 @@ Lines added: 1, lines removed: 4 -
Tool use: **code_edit** • Edit file: UpgradeSeqBatches.ps1 +
Tool use: **search_replace** • Edit file: UpgradeSeqBatches.ps1 **Chunk 1** Lines added: undefined, lines removed: 1 ```diff @@ -12,5 +12,4 @@ - Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan - Write-Host "Source .bat: $sourceFile" -ForegroundColor White -- Write-Host "Source .ps1: $sourcePs1" -ForegroundColor White - Write-Host "" + Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan + Write-Host "Source .bat: $sourceFile" -ForegroundColor White +- Write-Host "Source .ps1: $sourcePs1" -ForegroundColor White + Write-Host "" ``` @@ -907,56 +907,56 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: *** Add File: UpgradeSeqBatches.ps1 +# Copy the current UpdateSequences.bat to all subfolders that have it + -+$sourceFile = "R:\\Creative\\artsy\\maya\\0 ProjectStructure\\UpdateSequences.bat" ++$sourceFile = "R:\\Creative\\artsy\\maya\\0 ProjectStructure\\UpdateSequences.bat" + +if (-not (Test-Path $sourceFile)) { -+ Write-Error "Source file not found: $sourceFile" ++ Write-Error "Source file not found: $sourceFile" + exit 1 +} + -+Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan -+Write-Host "Source .bat: $sourceFile" -ForegroundColor White -+Write-Host "" ++Write-Host "=== Updating UpdateSequences.bat files ===" -ForegroundColor Cyan ++Write-Host "Source .bat: $sourceFile" -ForegroundColor White ++Write-Host "" + +# Find all UpdateSequences.bat files -+$targetFiles = Get-ChildItem -Path "R:\\Creative\\artsy\\maya" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } ++$targetFiles = Get-ChildItem -Path "R:\\Creative\\artsy\\maya" -Recurse -Filter "UpdateSequences.bat" | Where-Object { $_.FullName -ne $sourceFile } + -+Write-Host "Found $($targetFiles.Count) target files to update:" -ForegroundColor Yellow ++Write-Host "Found $($targetFiles.Count) target files to update:" -ForegroundColor Yellow +foreach ($file in $targetFiles) { -+ Write-Host " - $($file.FullName)" -ForegroundColor Gray ++ Write-Host " - $($file.FullName)" -ForegroundColor Gray +} -+Write-Host "" ++Write-Host "" + +$updatedCount = 0 +$failedCount = 0 + +foreach ($targetFile in $targetFiles) { + $targetDir = $targetFile.Directory.FullName -+ $targetBat = Join-Path $targetDir "UpdateSequences.bat" ++ $targetBat = Join-Path $targetDir "UpdateSequences.bat" + + try { + # Copy the .bat file + Copy-Item -Path $sourceFile -Destination $targetBat -Force -+ Write-Host "✓ Updated: $targetBat" -ForegroundColor Green ++ Write-Host "✓ Updated: $targetBat" -ForegroundColor Green + $updatedCount++ + } + catch { -+ Write-Host "✗ Failed to update: $targetBat" -ForegroundColor Red -+ Write-Host " Error: $($_.Exception.Message)" -ForegroundColor Red ++ Write-Host "✗ Failed to update: $targetBat" -ForegroundColor Red ++ Write-Host " Error: $($_.Exception.Message)" -ForegroundColor Red + $failedCount++ + } +} + -+Write-Host "" -+Write-Host "=== SUMMARY ===" -ForegroundColor Magenta -+Write-Host "Successfully updated: $updatedCount" -ForegroundColor Green -+Write-Host "Failed updates: $failedCount" -ForegroundColor Red -+Write-Host "Total targets: $($targetFiles.Count)" -ForegroundColor White ++Write-Host "" ++Write-Host "=== SUMMARY ===" -ForegroundColor Magenta ++Write-Host "Successfully updated: $updatedCount" -ForegroundColor Green ++Write-Host "Failed updates: $failedCount" -ForegroundColor Red ++Write-Host "Total targets: $($targetFiles.Count)" -ForegroundColor White + +if ($failedCount -eq 0) { -+ Write-Host "`n🎉 All UpdateSequences.bat files have been successfully updated!" -ForegroundColor Green ++ Write-Host "`n🎉 All UpdateSequences.bat files have been successfully updated!" -ForegroundColor Green +} else { -+ Write-Host "`n⚠️ Some files failed to update. Check the errors above." -ForegroundColor Yellow ++ Write-Host "`n⚠️ Some files failed to update. Check the errors above." -ForegroundColor Yellow +} + *** End Patch @@ -978,10 +978,10 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: +@echo off +setlocal EnableExtensions + -+set "ps1=R:\\Creative\\artsy\\maya\\0 ProjectStructure\\UpdateSequences.ps1" ++set "ps1=R:\\Creative\\artsy\\maya\\0 ProjectStructure\\UpdateSequences.ps1" +echo Running PowerShell update script... -+powershell -NoProfile -ExecutionPolicy Bypass -File "%ps1%" -+set "rc=%errorlevel%" ++powershell -NoProfile -ExecutionPolicy Bypass -File "%ps1%" ++set "rc=%errorlevel%" +echo PowerShell exited with RC=%rc% +echo( Done. Press any key to exit. +pause >nul @@ -1009,16 +1009,16 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: +) + +Set-StrictMode -Version Latest -+$ErrorActionPreference = 'Stop' ++$ErrorActionPreference = 'Stop' + +function Get-YoungestTimestamp { + param( + [Parameter(Mandatory)] [string]$FolderPath + ) + $files = Get-ChildItem -LiteralPath $FolderPath -Recurse -File -ErrorAction SilentlyContinue | -+ Where-Object { $_.FullName -notlike '*\_archive\*' } ++ Where-Object { $_.FullName -notlike '*\_archive\*' } + $young = $files | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -ExpandProperty LastWriteTime -+ if (-not $young) { return [datetime]::Parse('1900-01-01') } ++ if (-not $young) { return [datetime]::Parse('1900-01-01') } + return $young +} + @@ -1027,29 +1027,29 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + [Parameter(Mandatory)] [string]$SequenceFolderPath, + [Parameter(Mandatory)] [string]$SequenceName, + [string]$LogFile, -+ [string[]]$Extensions = @('.png','.jpg','.jpeg','.exr','.tif','.tiff','.bmp','.tga') ++ [string[]]$Extensions = @('.png','.jpg','.jpeg','.exr','.tif','.tiff','.bmp','.tga') + ) + $renamed = 0 + $collisions = 0 + $errors = 0 + $checked = 0 + -+ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME CHECK in '$SequenceFolderPath' (seq='$SequenceName')" | Add-Content -LiteralPath $LogFile } ++ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME CHECK in '$SequenceFolderPath' (seq='$SequenceName')" | Add-Content -LiteralPath $LogFile } + + $files = Get-ChildItem -LiteralPath $SequenceFolderPath -File -Recurse -ErrorAction SilentlyContinue | -+ Where-Object { $_.FullName -notlike '*\_archive\*' -and ($Extensions -contains $_.Extension.ToLower()) } ++ Where-Object { $_.FullName -notlike '*\_archive\*' -and ($Extensions -contains $_.Extension.ToLower()) } + foreach ($f in $files) { + $checked++ + $base = [System.IO.Path]::GetFileNameWithoutExtension($f.Name) + $ext = $f.Extension + $digits = $null + -+ if ($base -match '_(\d{6})$') { $digits = $Matches[1] } -+ elseif ($base -match '(?<!_)\b(\d{6})$') { $digits = $Matches[1] } -+ elseif ($base -match '(\d{4})$') { $digits = ('00' + $Matches[1]) } ++ if ($base -match '_(\d{6})$') { $digits = $Matches[1] } ++ elseif ($base -match '(?<!_)\b(\d{6})$') { $digits = $Matches[1] } ++ elseif ($base -match '(\d{4})$') { $digits = ('00' + $Matches[1]) } + else { continue } + -+ $targetBase = "$SequenceName" + '_' + $digits ++ $targetBase = "$SequenceName" + '_' + $digits + if ($base -eq $targetBase) { continue } + + $newName = $targetBase + $ext @@ -1057,16 +1057,16 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + try { + if (Test-Path -LiteralPath $newPath) { + $collisions++ -+ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME SKIP collision: '$($f.Name)' -> '$newName'" | Add-Content -LiteralPath $LogFile } ++ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME SKIP collision: '$($f.Name)' -> '$newName'" | Add-Content -LiteralPath $LogFile } + continue + } + Rename-Item -LiteralPath $f.FullName -NewName $newName -ErrorAction Stop + $renamed++ -+ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME: '$($f.Name)' -> '$newName'" | Add-Content -LiteralPath $LogFile } ++ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME: '$($f.Name)' -> '$newName'" | Add-Content -LiteralPath $LogFile } + } + catch { + $errors++ -+ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME ERROR for '$($f.Name)': $($_.Exception.Message)" | Add-Content -LiteralPath $LogFile } ++ if ($LogFile) { "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] RENAME ERROR for '$($f.Name)': $($_.Exception.Message)" | Add-Content -LiteralPath $LogFile } + } + } + @@ -1075,14 +1075,14 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + +try { + $root = (Get-Location).ProviderPath -+ $currentDir = Join-Path $root '_CURRENT' ++ $currentDir = Join-Path $root '_CURRENT' + if (-not (Test-Path -LiteralPath $currentDir)) { New-Item -ItemType Directory -Path $currentDir -Force | Out-Null } + -+ $logFile = Join-Path $currentDir '_UpdateSequences.log' -+ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) started in '$root' ===" | Add-Content -LiteralPath $logFile ++ $logFile = Join-Path $currentDir '_UpdateSequences.log' ++ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) started in '$root' ===" | Add-Content -LiteralPath $logFile + + # Scan for folders with YYYY-MM-DD format (e.g., 2025-08-27) -+ $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter '????-??-??' | Where-Object { $_.Name -ne '_archive' } ++ $dailyDirs = Get-ChildItem -LiteralPath $root -Directory -Filter '????-??-??' | Where-Object { $_.Name -ne '_archive' } + + $mapDaily = @{} + $dailiesScanned = 0 @@ -1091,20 +1091,20 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + $renameErrors = 0 + foreach ($d in $dailyDirs) { + $dailiesScanned++ -+ $seqDirs = @(Get-ChildItem -LiteralPath $d.FullName -Directory | Where-Object { $_.Name -ne '_archive' }) ++ $seqDirs = @(Get-ChildItem -LiteralPath $d.FullName -Directory | Where-Object { $_.Name -ne '_archive' }) + if ($seqDirs.Count -eq 0) { $seqDirs = @($d) } + foreach ($s in $seqDirs) { + if (-not (Test-Path -LiteralPath $s.FullName)) { continue } + $renameResult = Sync-SequenceFilenames -SequenceFolderPath $s.FullName -SequenceName $s.Name -LogFile $logFile + if ($DebugMode -or $renameResult.Renamed -gt 0 -or $renameResult.Collisions -gt 0 -or $renameResult.Errors -gt 0) { -+ Write-Host "[RENAME]|$($s.FullName)|$($s.Name)|checked=$($renameResult.Checked)|renamed=$($renameResult.Renamed)|collisions=$($renameResult.Collisions)|errors=$($renameResult.Errors)" -ForegroundColor Cyan ++ Write-Host "[RENAME]|$($s.FullName)|$($s.Name)|checked=$($renameResult.Checked)|renamed=$($renameResult.Renamed)|collisions=$($renameResult.Collisions)|errors=$($renameResult.Errors)" -ForegroundColor Cyan + } + $filesRenamedTotal += $renameResult.Renamed + $renameCollisions += $renameResult.Collisions + $renameErrors += $renameResult.Errors + + $young = Get-YoungestTimestamp -FolderPath $s.FullName -+ $filesCount = @(Get-ChildItem -LiteralPath $s.FullName -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.FullName -notlike '*\_archive\*' }).Count ++ $filesCount = @(Get-ChildItem -LiteralPath $s.FullName -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.FullName -notlike '*\_archive\*' }).Count + $seqName = $s.Name + $cand = [pscustomobject]@{ Seq=$seqName; SrcFull=$s.FullName; Young=$young; FilesCount=$filesCount } + if ($mapDaily.ContainsKey($seqName)) { @@ -1115,73 +1115,73 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + + $mapCurrent = @{} + if (Test-Path -LiteralPath $currentDir) { -+ foreach ($c in Get-ChildItem -LiteralPath $currentDir -Directory | Where-Object { $_.Name -ne '_archive' }) { ++ foreach ($c in Get-ChildItem -LiteralPath $currentDir -Directory | Where-Object { $_.Name -ne '_archive' }) { + $mapCurrent[$c.Name] = Get-YoungestTimestamp -FolderPath $c.FullName + } + } + + $lines = @() -+ $lines += "[MAPCOUNT]|count=$($mapDaily.Count)" ++ $lines += "[MAPCOUNT]|count=$($mapDaily.Count)" + $total = 0 + $toCopy = 0 + foreach ($k in $mapDaily.Keys) { + $total++ + $d = $mapDaily[$k] -+ $curYoung = if ($mapCurrent.ContainsKey($k)) { $mapCurrent[$k] } else { [datetime]::Parse('1900-01-01') } ++ $curYoung = if ($mapCurrent.ContainsKey($k)) { $mapCurrent[$k] } else { [datetime]::Parse('1900-01-01') } + if ($d.Young -gt $curYoung) { + $toCopy++ -+ $lines += "[COPY]|$($d.SrcFull)|$($d.Seq)|files=$($d.FilesCount)|young=$($d.Young.ToString('s'))" ++ $lines += "[COPY]|$($d.SrcFull)|$($d.Seq)|files=$($d.FilesCount)|young=$($d.Young.ToString('s'))" + } else { -+ $lines += "[SKIP]|$($d.SrcFull)|$($d.Seq)|reason=notNewer|srcYoung=$($d.Young.ToString('s'))|curYoung=$($curYoung.ToString('s'))" ++ $lines += "[SKIP]|$($d.SrcFull)|$($d.Seq)|reason=notNewer|srcYoung=$($d.Young.ToString('s'))|curYoung=$($curYoung.ToString('s'))" + } + } -+ $lines += "[META]|dailiesScanned=$dailiesScanned|sequencesTotal=$total|toCopy=$toCopy" ++ $lines += "[META]|dailiesScanned=$dailiesScanned|sequencesTotal=$total|toCopy=$toCopy" + -+ Write-Host "=== UPDATE PLAN ===" -ForegroundColor Cyan ++ Write-Host "=== UPDATE PLAN ===" -ForegroundColor Cyan + foreach ($line in $lines) { -+ if ($line -like '[COPY]*') { Write-Host $line -ForegroundColor Green } -+ elseif ($line -like '[SKIP]*') { Write-Host $line -ForegroundColor Yellow } ++ if ($line -like '[COPY]*') { Write-Host $line -ForegroundColor Green } ++ elseif ($line -like '[SKIP]*') { Write-Host $line -ForegroundColor Yellow } + else { Write-Host $line -ForegroundColor White } + } -+ Write-Host "==================" -ForegroundColor Cyan ++ Write-Host "==================" -ForegroundColor Cyan + + $sequencesMirrored = 0 + $mirrorFailures = 0 + foreach ($line in $lines) { -+ if ($line.StartsWith('[COPY]')) { -+ $parts = $line -split '\|' ++ if ($line.StartsWith('[COPY]')) { ++ $parts = $line -split '\|' + $srcFull = $parts[1] + $seqName = $parts[2] + $dstAbs = Join-Path $currentDir $seqName + if (-not (Test-Path -LiteralPath $dstAbs)) { New-Item -ItemType Directory -Path $dstAbs -Force | Out-Null } -+ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Mirror: '$srcFull' -> '$dstAbs'" | Add-Content -LiteralPath $logFile ++ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Mirror: '$srcFull' -> '$dstAbs'" | Add-Content -LiteralPath $logFile + $robocopyArgs = @( + $srcFull, + $dstAbs, -+ '/MIR','/MT:8','/R:1','/W:1','/COPY:DAT','/DCOPY:DAT','/FFT','/NFL','/NDL','/NP','/NJH','/NJS','/XD','_archive' ++ '/MIR','/MT:8','/R:1','/W:1','/COPY:DAT','/DCOPY:DAT','/FFT','/NFL','/NDL','/NP','/NJH','/NJS','/XD','_archive' + ) + $null = & robocopy @robocopyArgs 2>&1 | Add-Content -LiteralPath $logFile + $rc = $LASTEXITCODE -+ if ($rc -lt 8) { $sequencesMirrored++; "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] OK rc=$rc" | Add-Content -LiteralPath $logFile } -+ else { $mirrorFailures++; "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] ERROR rc=$rc" | Add-Content -LiteralPath $logFile } ++ if ($rc -lt 8) { $sequencesMirrored++; "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] OK rc=$rc" | Add-Content -LiteralPath $logFile } ++ else { $mirrorFailures++; "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] ERROR rc=$rc" | Add-Content -LiteralPath $logFile } + } + } + -+ Write-Host "=== SUMMARY REPORT ===" -ForegroundColor Magenta -+ Write-Host "Dailies scanned: $dailiesScanned" -ForegroundColor White -+ Write-Host "Sequences found: $total" -ForegroundColor White -+ Write-Host "Planned copies: $toCopy" -ForegroundColor Green -+ Write-Host "Completed OK: $sequencesMirrored" -ForegroundColor Green -+ Write-Host "Completed FAIL: $mirrorFailures" -ForegroundColor Red -+ Write-Host "Files renamed: $filesRenamedTotal (collisions: $renameCollisions, errors: $renameErrors)" -ForegroundColor White ++ Write-Host "=== SUMMARY REPORT ===" -ForegroundColor Magenta ++ Write-Host "Dailies scanned: $dailiesScanned" -ForegroundColor White ++ Write-Host "Sequences found: $total" -ForegroundColor White ++ Write-Host "Planned copies: $toCopy" -ForegroundColor Green ++ Write-Host "Completed OK: $sequencesMirrored" -ForegroundColor Green ++ Write-Host "Completed FAIL: $mirrorFailures" -ForegroundColor Red ++ Write-Host "Files renamed: $filesRenamedTotal (collisions: $renameCollisions, errors: $renameErrors)" -ForegroundColor White + + $skippedLines = @() -+ foreach ($line in $lines) { if ($line.StartsWith('[SKIP]')) { $skippedLines += $line } } ++ foreach ($line in $lines) { if ($line.StartsWith('[SKIP]')) { $skippedLines += $line } } + if ($skippedLines.Count -gt 0) { -+ Write-Host "`n=== SKIPPED SEQUENCES ===" -ForegroundColor Yellow ++ Write-Host "`n=== SKIPPED SEQUENCES ===" -ForegroundColor Yellow + $skippedByReason = @{} + foreach ($skip in $skippedLines) { -+ $parts = $skip -split '\|' ++ $parts = $skip -split '\|' + $reason = $parts[3] + $seqName = $parts[2] + if (-not $skippedByReason.ContainsKey($reason)) { $skippedByReason[$reason] = @() } @@ -1189,24 +1189,24 @@ I removed the conflicted files by mistake. I'll restore home-ready versions now: + } + foreach ($reason in $skippedByReason.Keys) { + $seqs = $skippedByReason[$reason] -+ Write-Host "$reason ($($seqs.Count) sequences):" -ForegroundColor Yellow -+ foreach ($seq in $seqs | Sort-Object) { Write-Host " - $seq" -ForegroundColor White } ++ Write-Host "$reason ($($seqs.Count) sequences):" -ForegroundColor Yellow ++ foreach ($seq in $seqs | Sort-Object) { Write-Host " - $seq" -ForegroundColor White } + } -+ Write-Host "========================" -ForegroundColor Yellow ++ Write-Host "========================" -ForegroundColor Yellow + } -+ Write-Host "=====================" -ForegroundColor Magenta ++ Write-Host "=====================" -ForegroundColor Magenta + -+ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) completed (d=$dailiesScanned seq=$total ok=$sequencesMirrored fail=$mirrorFailures) ===" | Add-Content -LiteralPath $logFile -+ Write-Host "@$logFile" ++ "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === UpdateSequences (ps1) completed (d=$dailiesScanned seq=$total ok=$sequencesMirrored fail=$mirrorFailures) ===" | Add-Content -LiteralPath $logFile ++ Write-Host "@$logFile" + exit 0 +} +catch { + $root = (Get-Location).ProviderPath -+ $currentDir = Join-Path $root '_CURRENT' ++ $currentDir = Join-Path $root '_CURRENT' + if (-not (Test-Path -LiteralPath $currentDir)) { New-Item -ItemType Directory -Path $currentDir -Force | Out-Null } -+ $planFile = Join-Path $currentDir '_UpdatePlan.txt' -+ ($_.Exception | Out-String) | Set-Content -LiteralPath ($planFile + '.error.txt') -Encoding ASCII -+ Write-Host "ERROR: $_" ++ $planFile = Join-Path $currentDir '_UpdatePlan.txt' ++ ($_.Exception | Out-String) | Set-Content -LiteralPath ($planFile + '.error.txt') -Encoding ASCII ++ Write-Host "ERROR: $_" + exit 1 +} + @@ -1387,7 +1387,7 @@ I'll remove the duplicate log echo from `UpdateSequences.bat` so only the PS1 pr *** Update File: UpdateSequences.bat @@ echo PowerShell exited with RC=%rc% --echo @"_CURRENT\_UpdateSequences.log" +-echo @"_CURRENT\_UpdateSequences.log" -echo Done. +echo Done. pause >nul diff --git a/.specstory/history/2025-10-30_00-40Z-update-gitignore-files-in-project-repos.md b/.specstory/history/2025-10-30_00-40Z-update-gitignore-files-in-project-repos.md index de83244..6c15146 100644 --- a/.specstory/history/2025-10-30_00-40Z-update-gitignore-files-in-project-repos.md +++ b/.specstory/history/2025-10-30_00-40Z-update-gitignore-files-in-project-repos.md @@ -32,7 +32,7 @@ Reviewing `components/gitignore` and scanning `R:\Creative\artsy\maya\` for YYYY
- Tool use: **list_directory** • Listed directory undefined • **0** results + Tool use: **list_dir** • Listed directory, 0 results No results found
@@ -127,20 +127,20 @@ Parameters: ```json { - "plan": "# Sync gitignore across project repositories\n\n## Overview\nUpdate `.gitignore` files in all project repositories matching the YYYY-MM-DD pattern in `R:\\Creative\\artsy\\maya\\` by merging missing lines from `components/gitignore`, then commit each change.\n\n## Steps\n\n### 1. Discover project directories\n- Scan `R:\\Creative\\artsy\\maya\\` for directories matching pattern `YYYY-MM-DD*` (e.g., `2025-08-02 Springtrap Prince of Egypt`)\n- Filter to only directories that are git repositories (contain `.git` folder)\n\n### 2. For each project repository:\n - **Verify git repo**: Check if `.git` exists\n - **Merge .gitignore**: \n - Use the merge logic from `UpgradeToGitProj.bat` (`:MergeTemplate` function)\n - If `.gitignore` doesn Jones exist, copy from template\n - If it exists, fastr `/x /c:\"%%L\" \"%dst%\" || (>>\"%dst白白\" echo %%L)` to append only missing lines\n - Source: `R:\\Creative\\artsy\\maya\\0 ProjectStructure\\components\\gitignore`\n - Destination: `{projectDir}\\.gitignore`\n - **Stage changes**: Run `git add .gitignore` in each repo\n - **Commit**: Run `git commit -m \"prproj gzip to LFS\"` in each repo\n\n### 3. Report results\n- List which repos were updated\n- List which repos had no changes (already up to date)\n- List any repos that failed (not git repos, permission errors, etc.)\n\n## Implementation notes\n- Use the same merge logic as `UpgradeToGitProj.bat` lines 248-250\n- Only process directories that are git repositories\n- Handle cases where `.gitignore` doesn't exist (copy entire template)\n- Commit only if there are actual changes to `.gitignore`\n", - "name": "Sync gitignore across project repos", - "overview": "Find all YYYY-MM-DD project directories in R:\\Creative\\artsy\\maya\\, merge changes from components/gitignore into each repo's .gitignore (using append-missing-lines logic), and commit with the specified message." + "plan": "# Sync gitignore across project repositories\n\n## Overview\nUpdate `.gitignore` files in all project repositories matching the YYYY-MM-DD pattern in `R:\\Creative\\artsy\\maya\\` by merging missing lines from `components/gitignore`, then commit each change.\n\n## Steps\n\n### 1. Discover project directories\n- Scan `R:\\Creative\\artsy\\maya\\` for directories matching pattern `YYYY-MM-DD*` (e.g., `2025-08-02 Springtrap Prince of Egypt`)\n- Filter to only directories that are git repositories (contain `.git` folder)\n\n### 2. For each project repository:\n - **Verify git repo**: Check if `.git` exists\n - **Merge .gitignore**: \n - Use the merge logic from `UpgradeToGitProj.bat` (`:MergeTemplate` function)\n - If `.gitignore` doesn Jones exist, copy from template\n - If it exists, fastr `/x /c:\"%%L\" \"%dst%\" || (>>\"%dst白白\" echo %%L)` to append only missing lines\n - Source: `R:\\Creative\\artsy\\maya\\0 ProjectStructure\\components\\gitignore`\n - Destination: `{projectDir}\\.gitignore`\n - **Stage changes**: Run `git add .gitignore` in each repo\n - **Commit**: Run `git commit -m \"prproj gzip to LFS\"` in each repo\n\n### 3. Report results\n- List which repos were updated\n- List which repos had no changes (already up to date)\n- List any repos that failed (not git repos, permission errors, etc.)\n\n## Implementation notes\n- Use the same merge logic as `UpgradeToGitProj.bat` lines 248-250\n- Only process directories that are git repositories\n- Handle cases where `.gitignore` doesn't exist (copy entire template)\n- Commit only if there are actual changes to `.gitignore`\n", + "name": "Sync gitignore across project repos", + "overview": "Find all YYYY-MM-DD project directories in R:\\Creative\\artsy\\maya\\, merge changes from components/gitignore into each repo's .gitignore (using append-missing-lines logic), and commit with the specified message." } ``` Additional data: ```json { - "status": "pending", - "reviewData": { - "status": "Requested", - "selectedOption": "none", - "isShowingInput": false + "status": "pending", + "reviewData": { + "status": "Requested", + "selectedOption": "none", + "isShowingInput": false } } ``` @@ -148,7 +148,7 @@ Result: ```json { - "rejected": {} + "rejected": {} } ``` Status: **completed** @@ -181,20 +181,20 @@ Parameters: ```json { - "plan": "# Sync gitignore across project repositories\n\n## Overview\n\nUpdate `.gitignore` files in all project repositories matching the YYYY-MM-DD pattern in `R:\\Creative\\artsy\\maya\\` by merging missing lines from `components/gitignore`, then commit each change.\n\n## Steps\n\n### 1. Discover project directories\n\n- Scan `R:\\Creative\\artsy\\maya\\` for directories matching pattern `YYYY-MM-DD*` (e.g., `2025-08-02 Springtrap Prince of Egypt`)\n- Filter to only directories that are git repositories (contain `.git` folder)\n\n### 2. For each project repository:\n\n- **Verify git repo**: Check if `.git` exists\n- **Merge .gitignore**: \n - Use the merge logic from `UpgradeToGitProj.bat` (`:MergeTemplate` function)\n - If `.gitignore` doesn't exist, copy the template file in full\n - If it already exists, iterate each template line and append only those lines not present (e.g., `findstr /x /c:\"%%L\" \"%dst%\" || (>>\"%dst%\" echo %%L)`)\n - Source: `R:\\Creative\\artsy\\maya\\0 ProjectStructure\\components\\gitignore`\n - Destination: `{projectDir}\\.gitignore`\n- **Stage changes**: Run `git add .gitignore` in each repo\n- **Commit**: Run `git commit -m \"prproj gzip to LFS\"` in each repo\n\n### 3. Report results\n\n- List which repos were updated\n- List which repos had no changes (already up to date)\n- List any repos that failed (not git repos, permission errors, etc.)\n\n## Implementation notes\n\n- Use the same merge logic as `UpgradeToGitProj.bat` lines 248-250\n- Only process directories that are git repositories\n- Handle cases where `.gitignore` doesn't exist (copy entire template)\n- Commit only if there are actual changes to `.gitignore`", - "name": "Sync gitignore across project repos", - "overview": "Find all YYYY-MM-DD project directories in R:\\Creative\\artsy\\maya\\, merge changes from components/gitignore into each repo's .gitignore (using append-missing-lines logic), and commit with the specified message." + "plan": "# Sync gitignore across project repositories\n\n## Overview\n\nUpdate `.gitignore` files in all project repositories matching the YYYY-MM-DD pattern in `R:\\Creative\\artsy\\maya\\` by merging missing lines from `components/gitignore`, then commit each change.\n\n## Steps\n\n### 1. Discover project directories\n\n- Scan `R:\\Creative\\artsy\\maya\\` for directories matching pattern `YYYY-MM-DD*` (e.g., `2025-08-02 Springtrap Prince of Egypt`)\n- Filter to only directories that are git repositories (contain `.git` folder)\n\n### 2. For each project repository:\n\n- **Verify git repo**: Check if `.git` exists\n- **Merge .gitignore**: \n - Use the merge logic from `UpgradeToGitProj.bat` (`:MergeTemplate` function)\n - If `.gitignore` doesn't exist, copy the template file in full\n - If it already exists, iterate each template line and append only those lines not present (e.g., `findstr /x /c:\"%%L\" \"%dst%\" || (>>\"%dst%\" echo %%L)`)\n - Source: `R:\\Creative\\artsy\\maya\\0 ProjectStructure\\components\\gitignore`\n - Destination: `{projectDir}\\.gitignore`\n- **Stage changes**: Run `git add .gitignore` in each repo\n- **Commit**: Run `git commit -m \"prproj gzip to LFS\"` in each repo\n\n### 3. Report results\n\n- List which repos were updated\n- List which repos had no changes (already up to date)\n- List any repos that failed (not git repos, permission errors, etc.)\n\n## Implementation notes\n\n- Use the same merge logic as `UpgradeToGitProj.bat` lines 248-250\n- Only process directories that are git repositories\n- Handle cases where `.gitignore` doesn't exist (copy entire template)\n- Commit only if there are actual changes to `.gitignore`", + "name": "Sync gitignore across project repos", + "overview": "Find all YYYY-MM-DD project directories in R:\\Creative\\artsy\\maya\\, merge changes from components/gitignore into each repo's .gitignore (using append-missing-lines logic), and commit with the specified message." } ``` Additional data: ```json { - "status": "accepted", - "reviewData": { - "status": "Done", - "selectedOption": "approve", - "isShowingInput": false + "status": "accepted", + "reviewData": { + "status": "Done", + "selectedOption": "approve", + "isShowingInput": false } } ``` @@ -202,7 +202,7 @@ Result: ```json { - "accepted": {} + "accepted": {} } ``` Status: **completed** @@ -228,7 +228,7 @@ _**Agent (model gpt-5-codex, mode Agent)**_
- Tool use: **list_directory** • Listed directory undefined • **0** results + Tool use: **list_dir** • Listed directory, 0 results No results found
@@ -243,7 +243,7 @@ No results found
- Tool use: **list_directory** • Listed directory undefined • **0** results + Tool use: **list_dir** • Listed directory, 0 results No results found
@@ -304,7 +304,7 @@ _**Agent (model gpt-5-codex, mode Agent)**_ --- -
Tool use: **command** • Run command: cd R:\Creative\artsy\maya +
Tool use: **run_terminal_cmd** • Run command: cd R:\Creative\artsy\maya Get-ChildItem -Directory | Where-Object { $_.Name -match '^[0-9]{4}-[0-9]{2}-[0-9]{2}' -and (Test-Path (Join-Path $_.FullName '.git')) } | Select-Object -ExpandProperty FullName ```bash @@ -333,7 +333,7 @@ I'm drafting a PowerShell script to sync a template `.gitignore` to multiple git --- -
Tool use: **command** • Run command: cd R:\Creative\artsy\maya +
Tool use: **run_terminal_cmd** • Run command: cd R:\Creative\artsy\maya $ErrorActionPreference = 'Stop' $tplPath = 'R:\Creative\artsy\maya\0 ProjectStructure\components\gitignore' $repos = @( @@ -458,7 +458,7 @@ $summary --- -
Tool use: **command** • Run command: cd R:\Creative\artsy\maya +
Tool use: **run_terminal_cmd** • Run command: cd R:\Creative\artsy\maya $repos = @( 'R:\Creative\artsy\maya\2025-07-09 LolbitNSFW', 'R:\Creative\artsy\maya\2025-07-11 GothFexa', @@ -526,7 +526,7 @@ foreach ($repo in $repos) { --- -
Tool use: **command** • Run command: cd "R:\Creative\artsy\maya\2025-07-29 Springtrap_Goopie" +
Tool use: **run_terminal_cmd** • Run command: cd "R:\Creative\artsy\maya\2025-07-29 Springtrap_Goopie" git log -1 --pretty=%s ```bash @@ -551,7 +551,7 @@ I’m figuring out how to identify which repos actually got updated by checking --- -
Tool use: **command** • Run command: $repos = @( +
Tool use: **run_terminal_cmd** • Run command: $repos = @( 'R:\Creative\artsy\maya\2025-07-09 LolbitNSFW', 'R:\Creative\artsy\maya\2025-07-11 GothFexa', 'R:\Creative\artsy\maya\2025-07-29 Springtrap_Goopie', @@ -627,7 +627,7 @@ spawn c:\Users\Nathan\AppData\Local\Programs\cursor\resources\app\node_modules\@ --- -
Tool use: **command** • Run command: $repos = @( +
Tool use: **run_terminal_cmd** • Run command: $repos = @( 'R:\Creative\artsy\maya\2025-07-09 LolbitNSFW', 'R:\Creative\artsy\maya\2025-07-11 GothFexa', 'R:\Creative\artsy\maya\2025-07-29 Springtrap_Goopie', diff --git a/.specstory/history/2025-11-12_01-02Z-compression-failure-and-memory-allocation-issues.md b/.specstory/history/2025-11-12_01-02Z-compression-failure-and-memory-allocation-issues.md index c7d3ac4..8cf9df0 100644 --- a/.specstory/history/2025-11-12_01-02Z-compression-failure-and-memory-allocation-issues.md +++ b/.specstory/history/2025-11-12_01-02Z-compression-failure-and-memory-allocation-issues.md @@ -302,20 +302,20 @@ Parameters: ```json { - "plan": "# Implement Size-Aware RAM Estimation and Bin-Packing for 7z Workers\n\n## Overview\nReplace the fixed 3GB per-job RAM estimate with dynamic estimation based on actual sequence sizes. Use bin-packing (largest-first) to determine optimal concurrent workers when `Max7zInst` is 0.\n\n## Problem Analysis\n\n**Current Issues:**\n1. Fixed 3GB per-job estimate doesn't account for actual sequence sizes\n2. A 3.6GB sequence with 1GB dictionary needs ~12-13GB RAM (not 3GB)\n3. 7z dictionary overhead: ~11x dictionary size (1GB dict ≈ 11GB RAM for dictionary operations)\n4. No consideration of sequence sizes when calculating workers\n\n**Failure Case:**\n- 22 workers × 3GB = 66GB allocated\n- But a 3.6GB sequence actually needs ~12GB RAM\n- Result: \"Can't allocate required memory!\" error\n\n## Implementation Plan\n\n### 1. Fix `estimate_ram_per_job()` function\n**File:** `zip_sequences.py` (lines 186-220)\n\nUpdate the RAM estimation formula for 7z:\n- **Dictionary RAM**: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\n- **Input buffer**: `min(sequence_size * 0.15, 2GB)` - 7z processes in chunks, doesn't need full file in RAM\n- **Overhead**: ~1GB for compression overhead\n- **Formula**: `11GB (dict) + min(sequence_size * 0.15, 2GB) + 1GB (overhead)`\n\nExamples:\n- 3.6GB sequence: 11GB + 540MB + 1GB = ~12.5GB\n- 25GB sequence: 11GB + 2GB + 1GB = 14GB\n- 500MB sequence: 11GB + 75MB + 1GB = ~12GB (minimum)\n\n### 2. Implement bin-packing algorithm in `max_workers()`\n**File:** `zip_sequences.py` (lines 240-339)\n\nWhen `Max7zInst` is 0 (auto-calculate):\n1. **Estimate RAM for each work item** using `estimate_ram_per_job()`\n2. **Sort work items by estimated RAM** (largest first)\n3. **Bin-packing algorithm**:\n - Start with available RAM (80% of total, then 95% for compression)\n - For each work item (largest first):\n - If it fits in current \"bin\" (available RAM), add it\n - If not, start a new bin (increment worker count)\n - Track remaining RAM in current bin\n - Return: (worker_count, fixed_dict_size_bytes)\n\n4. **Variable RAM per job**: Each job gets its estimated RAM requirement\n - Store estimated RAM per work item\n - Pass individual RAM limits to each job\n\n### 3. Update `run_zip()` to handle variable RAM per job\n**File:** `zip_sequences.py` (lines 656-750)\n\nCurrently passes single `per_job_memory_limit` to all jobs. Change to:\n- `max_workers()` returns: `(worker_count, ram_limits_dict)` where `ram_limits_dict` maps `seq_dir` → estimated RAM\n- `process_zip()` receives individual RAM limit for its sequence\n- `zip_sequence()` uses the per-job RAM limit (but still uses fixed 1GB dictionary)\n\n### 4. Update `zip_sequence()` to use per-job RAM limit\n**File:** `zip_sequences.py` (lines 459-560)\n\nThe function already accepts `per_job_memory_limit` but treats it as dictionary size. We need to:\n- Keep dictionary at 1GB (fixed)\n- The RAM limit is informational/logging only (7z doesn't have a direct RAM limit flag)\n- Or: Calculate appropriate dictionary size based on available RAM per job (but user wants 1GB dict)\n\n**Note:** 7z doesn't have a `-mmem` flag. Memory is controlled via dictionary size. Since user wants 1GB dictionary, we can't directly limit RAM per job. However, we can:\n- Use the RAM estimates to determine how many jobs can run concurrently\n- Keep dictionary at 1GB\n- The bin-packing ensures we don't exceed available RAM\n\n### 5. Update verbose logging\n**File:** `zip_sequences.py`\n\nShow:\n- Individual RAM estimates per sequence\n- Bin-packing results (which sequences in which bins)\n- Total RAM allocation\n\n## Files to Modify\n\n1. `zip_sequences.py`:\n - `estimate_ram_per_job()`: Update formula (lines 186-220)\n - `max_workers()`: Add bin-packing logic (lines 240-339)\n - `run_zip()`: Handle variable RAM limits (lines 656-750)\n - `process_zip()`: Accept individual RAM limit (line 644)\n - `zip_sequence()`: Keep dictionary at 1GB (lines 459-560)\n\n## Testing Considerations\n\n- Test with small sequences (<1GB)\n- Test with medium sequences (3-5GB) - should need ~12-13GB RAM each\n- Test with large sequences (25GB+) - should need ~14GB RAM each\n- Test bin-packing with mixed sizes\n- Verify dictionary stays at 1GB for all jobs", - "name": "Impl", - "overview": "Replace the fixed 3GB per-job estimate with size-aware RAM estimation that accounts for dictionary overhead (~11x dictionary size), input buffers, and compression overhead. Implement bin-packing algorithm (largest-first) to determine optimal concurrent workers when Max7zInst is 0." + "plan": "# Implement Size-Aware RAM Estimation and Bin-Packing for 7z Workers\n\n## Overview\nReplace the fixed 3GB per-job RAM estimate with dynamic estimation based on actual sequence sizes. Use bin-packing (largest-first) to determine optimal concurrent workers when `Max7zInst` is 0.\n\n## Problem Analysis\n\n**Current Issues:**\n1. Fixed 3GB per-job estimate doesn't account for actual sequence sizes\n2. A 3.6GB sequence with 1GB dictionary needs ~12-13GB RAM (not 3GB)\n3. 7z dictionary overhead: ~11x dictionary size (1GB dict ≈ 11GB RAM for dictionary operations)\n4. No consideration of sequence sizes when calculating workers\n\n**Failure Case:**\n- 22 workers × 3GB = 66GB allocated\n- But a 3.6GB sequence actually needs ~12GB RAM\n- Result: \"Can't allocate required memory!\" error\n\n## Implementation Plan\n\n### 1. Fix `estimate_ram_per_job()` function\n**File:** `zip_sequences.py` (lines 186-220)\n\nUpdate the RAM estimation formula for 7z:\n- **Dictionary RAM**: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\n- **Input buffer**: `min(sequence_size * 0.15, 2GB)` - 7z processes in chunks, doesn't need full file in RAM\n- **Overhead**: ~1GB for compression overhead\n- **Formula**: `11GB (dict) + min(sequence_size * 0.15, 2GB) + 1GB (overhead)`\n\nExamples:\n- 3.6GB sequence: 11GB + 540MB + 1GB = ~12.5GB\n- 25GB sequence: 11GB + 2GB + 1GB = 14GB\n- 500MB sequence: 11GB + 75MB + 1GB = ~12GB (minimum)\n\n### 2. Implement bin-packing algorithm in `max_workers()`\n**File:** `zip_sequences.py` (lines 240-339)\n\nWhen `Max7zInst` is 0 (auto-calculate):\n1. **Estimate RAM for each work item** using `estimate_ram_per_job()`\n2. **Sort work items by estimated RAM** (largest first)\n3. **Bin-packing algorithm**:\n - Start with available RAM (80% of total, then 95% for compression)\n - For each work item (largest first):\n - If it fits in current \"bin\" (available RAM), add it\n - If not, start a new bin (increment worker count)\n - Track remaining RAM in current bin\n - Return: (worker_count, fixed_dict_size_bytes)\n\n4. **Variable RAM per job**: Each job gets its estimated RAM requirement\n - Store estimated RAM per work item\n - Pass individual RAM limits to each job\n\n### 3. Update `run_zip()` to handle variable RAM per job\n**File:** `zip_sequences.py` (lines 656-750)\n\nCurrently passes single `per_job_memory_limit` to all jobs. Change to:\n- `max_workers()` returns: `(worker_count, ram_limits_dict)` where `ram_limits_dict` maps `seq_dir` → estimated RAM\n- `process_zip()` receives individual RAM limit for its sequence\n- `zip_sequence()` uses the per-job RAM limit (but still uses fixed 1GB dictionary)\n\n### 4. Update `zip_sequence()` to use per-job RAM limit\n**File:** `zip_sequences.py` (lines 459-560)\n\nThe function already accepts `per_job_memory_limit` but treats it as dictionary size. We need to:\n- Keep dictionary at 1GB (fixed)\n- The RAM limit is informational/logging only (7z doesn't have a direct RAM limit flag)\n- Or: Calculate appropriate dictionary size based on available RAM per job (but user wants 1GB dict)\n\n**Note:** 7z doesn't have a `-mmem` flag. Memory is controlled via dictionary size. Since user wants 1GB dictionary, we can't directly limit RAM per job. However, we can:\n- Use the RAM estimates to determine how many jobs can run concurrently\n- Keep dictionary at 1GB\n- The bin-packing ensures we don't exceed available RAM\n\n### 5. Update verbose logging\n**File:** `zip_sequences.py`\n\nShow:\n- Individual RAM estimates per sequence\n- Bin-packing results (which sequences in which bins)\n- Total RAM allocation\n\n## Files to Modify\n\n1. `zip_sequences.py`:\n - `estimate_ram_per_job()`: Update formula (lines 186-220)\n - `max_workers()`: Add bin-packing logic (lines 240-339)\n - `run_zip()`: Handle variable RAM limits (lines 656-750)\n - `process_zip()`: Accept individual RAM limit (line 644)\n - `zip_sequence()`: Keep dictionary at 1GB (lines 459-560)\n\n## Testing Considerations\n\n- Test with small sequences (<1GB)\n- Test with medium sequences (3-5GB) - should need ~12-13GB RAM each\n- Test with large sequences (25GB+) - should need ~14GB RAM each\n- Test bin-packing with mixed sizes\n- Verify dictionary stays at 1GB for all jobs", + "name": "Impl", + "overview": "Replace the fixed 3GB per-job estimate with size-aware RAM estimation that accounts for dictionary overhead (~11x dictionary size), input buffers, and compression overhead. Implement bin-packing algorithm (largest-first) to determine optimal concurrent workers when Max7zInst is 0." } ``` Additional data: ```json { - "status": "accepted", - "reviewData": { - "status": "Done", - "selectedOption": "approve", - "isShowingInput": false + "status": "accepted", + "reviewData": { + "status": "Done", + "selectedOption": "approve", + "isShowingInput": false } } ``` @@ -323,7 +323,7 @@ Result: ```json { - "accepted": {} + "accepted": {} } ``` Status: **completed** @@ -412,7 +412,7 @@ Implementing the changes step by step: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 14, lines removed: 13 @@ -420,7 +420,7 @@ Lines added: 14, lines removed: 13 ```diff @@ -198,20 +198,21 @@ - if ZIPPER_TYPE == "7z": + if ZIPPER_TYPE == "7z": - # Base RAM: 500MB per job - base_ram = 500 * 1024 * 1024 # 500 MB + # Fixed dictionary size: 1GB (1024MB) @@ -428,7 +428,7 @@ Lines added: 14, lines removed: 13 + FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024 - # Compression factor: 7z can use significant RAM, especially for large files -- # Use 0.15x factor (conservative estimate accounting for 7z's 80% usage) +- # Use 0.15x factor (conservative estimate accounting for 7z's 80% usage) - compression_factor = 0.15 + # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations) + # This is the main memory consumer for 7z LZMA compression @@ -437,7 +437,7 @@ Lines added: 14, lines removed: 13 - # For very large folders (>10GB), cap at 8GB per job - max_ram_per_job = 8 * 1024 * 1024 * 1024 # 8 GB - large_folder_threshold = 10 * 1024 * 1024 * 1024 # 10 GB -+ # Input buffer: 7z processes in chunks, doesn't need full file in RAM ++ # Input buffer: 7z processes in chunks, doesn't need full file in RAM + # Use 15% of sequence size, capped at 2GB + input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024) @@ -487,7 +487,7 @@ Implementing the bin-packing algorithm in `max_workers()`: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 68, lines removed: 16 @@ -496,7 +496,7 @@ Lines added: 68, lines removed: 16 @@ -286,40 +286,92 @@ # Auto-calculate based on RAM if Max7zInst not configured -- # "Balls-to-the-walls" mode: use maximum resources +- # "Balls-to-the-walls" mode: use maximum resources + # Use bin-packing algorithm with size-aware RAM estimation if available_ram is not None: - # 7z uses ~2-3x dictionary size in RAM, use 3x for aggressive mode @@ -542,7 +542,7 @@ Lines added: 68, lines removed: 16 + placed = True + break + -+ # If doesn't fit, create new bin ++ # If doesn't fit, create new bin + if not placed: + bins.append([(seq_dir, estimated_ram, work_item)]) + bin_remaining.append(compression_ram - estimated_ram) @@ -550,7 +550,7 @@ Lines added: 68, lines removed: 16 + # Worker count is number of bins + worker_count = len(bins) - # Cap at number of actual work items (can't have more workers than jobs) + # Cap at number of actual work items (can't have more workers than jobs) num_work_items = len(work_items) if work_items else 0 if num_work_items > 0: - ram_limit = min(ram_limit, num_work_items) @@ -558,7 +558,7 @@ Lines added: 68, lines removed: 16 - # Use RAM limit directly (no CPU limit) - final_limit = ram_limit -+ # Respect user's --jobs if provided ++ # Respect user's --jobs if provided if requested and requested > 0: - final_limit = min(final_limit, requested) + worker_count = min(worker_count, requested) @@ -569,30 +569,30 @@ Lines added: 68, lines removed: 16 - ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) + total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3) log( - "zip", -- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -- f"RAM limit: {ram_limit}, work items: {num_work_items}, requested: {requested}, final: {final_limit}", -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)", + "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " +- f"RAM limit: {ram_limit}, work items: {num_work_items}, requested: {requested}, final: {final_limit}", ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)", + verbose_only=True, + verbose=verbose + ) + log( -+ "zip", -+ f"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences", ++ "zip", ++ f"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences", + verbose_only=True, + verbose=verbose + ) + if len(bins) > 0: + bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins] + log( -+ "zip", -+ f"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}", ++ "zip", ++ f"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}", + verbose_only=True, + verbose=verbose + ) + log( -+ "zip", -+ f"Final worker count: {worker_count} (requested: {requested})", ++ "zip", ++ f"Final worker count: {worker_count} (requested: {requested})", verbose_only=True, verbose=verbose ) @@ -622,7 +622,7 @@ Updating the other return paths in `max_workers()` to return a dict, and updatin -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 6, lines removed: undefined @@ -662,7 +662,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 8, lines removed: undefined @@ -718,7 +718,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -786,7 +786,7 @@ Updating `zip_sequence()` to keep the dictionary at 1GB fixed, regardless of the -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 5, lines removed: 5 @@ -799,12 +799,12 @@ Lines added: 5, lines removed: 5 - if per_job_memory_limit is not None: - # per_job_memory_limit is actually the fixed dictionary size in bytes - dict_size_mb = per_job_memory_limit // (1024 * 1024) -- cmd.append(f"-md={dict_size_mb}m") +- cmd.append(f"-md={dict_size_mb}m") + # Always use fixed dictionary size: 1GB (1024MB) + # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only) + # We keep dictionary at 1GB for best compression regardless of RAM estimate + FIXED_DICT_SIZE_MB = 1024 -+ cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") ++ cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") cmd.extend([ ``` @@ -859,7 +859,7 @@ Updating the zip compression path to return a dict: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 6, lines removed: 2 @@ -921,7 +921,7 @@ Model supplied an ambiguous edit -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -929,7 +929,7 @@ Lines added: 1, lines removed: 1 ```diff @@ -256,5 +256,5 @@ if verbose: - log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) + log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) - return (cpu_limit, None) + return (cpu_limit, {}) @@ -979,7 +979,7 @@ Fixing the remaining return statement: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1354,36 +1354,36 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "3dc966ec-c9bc-4a5a-8879-956d2e9c1b38" + "codeblockId": "3dc966ec-c9bc-4a5a-8879-956d2e9c1b38" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " }\r\n \r\n+ \r\n+ def log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n+ if verbose_only and not verbose:\r\n+ return\r\n+ print(f\"[{mode}] {message}\", flush=True)\r\n+ \r\n \r\n def load_config() -> dict:\r", - "oldStart": 58, - "newStart": 58, - "oldLines": 4, - "newLines": 10, - "linesAdded": 6 + "diffString": " }\r\n \r\n+ \r\n+ def log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n+ if verbose_only and not verbose:\r\n+ return\r\n+ print(f\"[{mode}] {message}\", flush=True)\r\n+ \r\n \r\n def load_config() -> dict:\r", + "oldStart": 58, + "newStart": 58, + "oldLines": 4, + "newLines": 10, + "linesAdded": 6 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n return merged\r\n except json.JSONDecodeError:\r\n continue\r\n except OSError:\r\n continue\r\n \r\n # If no config found, return defaults\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\")\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n return merged\r\n except json.JSONDecodeError:\r\n continue\r\n except OSError:\r\n continue\r\n \r\n # If no config found, return defaults\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\")\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1409,36 +1409,36 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "1dcd5514-d762-4a9a-97d1-e93fce4a9af3" + "codeblockId": "1dcd5514-d762-4a9a-97d1-e93fce4a9af3" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " return (final_limit, ram_limits_dict)\r\n \r\n- \r\n- def log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n- if verbose_only and not verbose:\r\n- return\r\n- print(f\"[{mode}] {message}\")\r\n- \r\n \r\n def is_archive_path(path: Path) -> bool:\r", - "oldStart": 448, - "newStart": 448, - "oldLines": 10, - "newLines": 4, - "linesRemoved": 6 + "diffString": " return (final_limit, ram_limits_dict)\r\n \r\n- \r\n- def log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n- if verbose_only and not verbose:\r\n- return\r\n- print(f\"[{mode}] {message}\")\r\n- \r\n \r\n def is_archive_path(path: Path) -> bool:\r", + "oldStart": 448, + "newStart": 448, + "oldLines": 10, + "newLines": 4, + "linesRemoved": 6 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n return merged\r\n except json.JSONDecodeError:\r\n continue\r\n except OSError:\r\n continue\r\n \r\n # If no config found, return defaults\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n return merged\r\n except json.JSONDecodeError:\r\n continue\r\n except OSError:\r\n continue\r\n \r\n # If no config found, return defaults\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1465,44 +1465,44 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "216b131a-3043-4fe5-b7bc-4ca53001c628" + "codeblockId": "216b131a-3043-4fe5-b7bc-4ca53001c628" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " ]\r\n \r\n+ log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n+ log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r", - "oldStart": 77, - "newStart": 77, - "oldLines": 7, - "newLines": 9, - "linesAdded": 2 + "diffString": " ]\r\n \r\n+ log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n+ log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r", + "oldStart": 77, + "newStart": 77, + "oldLines": 7, + "newLines": 9, + "linesAdded": 2 }, { - "diffString": " merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n+ log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n+ log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n+ log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n+ log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n \r", - "oldStart": 86, - "newStart": 88, - "oldLines": 11, - "newLines": 15, - "linesAdded": 4 + "diffString": " merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n+ log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n+ log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n+ log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n+ log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n \r", + "oldStart": 86, + "newStart": 88, + "oldLines": 11, + "newLines": 15, + "linesAdded": 4 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1529,36 +1529,36 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "c11fb9a3-9bcc-4e99-a49f-f1911dd6d57a" + "codeblockId": "c11fb9a3-9bcc-4e99-a49f-f1911dd6d57a" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " def main() -> int:\r\n args = parse_args()\r\n+ log(\"init\", \"zip_sequences starting up...\")\r\n+ log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n+ log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n+ if ZIPPER_TYPE == \"7z\":\r\n+ exe = SEVEN_Z_EXE or \"not found\"\r\n+ max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n+ log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n \r\n if args.mode == \"expand\":\r", - "oldStart": 929, - "newStart": 929, - "oldLines": 4, - "newLines": 11, - "linesAdded": 7 + "diffString": " def main() -> int:\r\n args = parse_args()\r\n+ log(\"init\", \"zip_sequences starting up...\")\r\n+ log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n+ log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n+ if ZIPPER_TYPE == \"7z\":\r\n+ exe = SEVEN_Z_EXE or \"not found\"\r\n+ max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n+ log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n \r\n if args.mode == \"expand\":\r", + "oldStart": 929, + "newStart": 929, + "oldLines": 4, + "newLines": 11, + "linesAdded": 7 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1615,68 +1615,68 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "ad47bbd1-38ad-4c77-bb5f-00b30a00f45c" + "codeblockId": "ad47bbd1-38ad-4c77-bb5f-00b30a00f45c" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " def run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n+ log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n+ total_scanned = 0\r\n+ quick_skipped = 0\r\n+ state_skipped = 0\r\n+ empty_dirs = 0\r\n+ queued = 0\r\n \r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n+ total_scanned += 1\r\n+ rel = seq_dir.relative_to(RENDER_ROOT)\r\n+ if total_scanned <= 5 or total_scanned % 10 == 0:\r\n+ log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n+ \r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r", - "oldStart": 733, - "newStart": 733, - "oldLines": 7, - "newLines": 18, - "linesAdded": 11 + "diffString": " def run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n+ log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n+ total_scanned = 0\r\n+ quick_skipped = 0\r\n+ state_skipped = 0\r\n+ empty_dirs = 0\r\n+ queued = 0\r\n \r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n+ total_scanned += 1\r\n+ rel = seq_dir.relative_to(RENDER_ROOT)\r\n+ if total_scanned <= 5 or total_scanned % 10 == 0:\r\n+ log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n+ \r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r", + "oldStart": 733, + "newStart": 733, + "oldLines": 7, + "newLines": 18, + "linesAdded": 11 }, { - "diffString": " # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n+ quick_skipped += 1\r\n+ if quick_skipped <= 5:\r\n+ log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r", - "oldStart": 763, - "newStart": 774, - "oldLines": 4, - "newLines": 7, - "linesAdded": 3 + "diffString": " # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n+ quick_skipped += 1\r\n+ if quick_skipped <= 5:\r\n+ log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r", + "oldStart": 763, + "newStart": 774, + "oldLines": 4, + "newLines": 7, + "linesAdded": 3 }, { - "diffString": " seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n+ empty_dirs += 1\r\n+ if empty_dirs <= 5:\r\n+ log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n+ state_skipped += 1\r\n+ if state_skipped <= 5:\r\n+ log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r", - "oldStart": 777, - "newStart": 791, - "oldLines": 9, - "newLines": 15, - "linesAdded": 6 + "diffString": " seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n+ empty_dirs += 1\r\n+ if empty_dirs <= 5:\r\n+ log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n+ state_skipped += 1\r\n+ if state_skipped <= 5:\r\n+ log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r", + "oldStart": 777, + "newStart": 791, + "oldLines": 9, + "newLines": 15, + "linesAdded": 6 }, { - "diffString": " \r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n+ queued += 1\r\n+ if queued <= 5 or queued % 5 == 0:\r\n+ total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n+ size_gb = total_bytes / (1024 ** 3)\r\n+ log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n \r\n if not work_items:\r", - "oldStart": 798, - "newStart": 818, - "oldLines": 4, - "newLines": 9, - "linesAdded": 5 + "diffString": " \r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n+ queued += 1\r\n+ if queued <= 5 or queued % 5 == 0:\r\n+ total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n+ size_gb = total_bytes / (1024 ** 3)\r\n+ log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n \r\n if not work_items:\r", + "oldStart": 798, + "newStart": 818, + "oldLines": 4, + "newLines": 9, + "linesAdded": 5 }, { - "diffString": " else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n+ log(\r\n+ \"scan\",\r\n+ f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n+ f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n+ )\r\n return 0\r\n \r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n+ log(\r\n+ \"init\",\r\n+ f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n+ )\r\n \r\n updated_paths: list[Path] = []\r", - "oldStart": 804, - "newStart": 829, - "oldLines": 8, - "newLines": 17, - "linesAdded": 9 + "diffString": " else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n+ log(\r\n+ \"scan\",\r\n+ f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n+ f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n+ )\r\n return 0\r\n \r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n+ log(\r\n+ \"init\",\r\n+ f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n+ )\r\n \r\n updated_paths: list[Path] = []\r", + "oldStart": 804, + "newStart": 829, + "oldLines": 8, + "newLines": 17, + "linesAdded": 9 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n total_scanned = 0\r\n quick_skipped = 0\r\n state_skipped = 0\r\n empty_dirs = 0\r\n queued = 0\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n total_scanned += 1\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n if total_scanned <= 5 or total_scanned % 10 == 0:\r\n log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n quick_skipped += 1\r\n if quick_skipped <= 5:\r\n log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n empty_dirs += 1\r\n if empty_dirs <= 5:\r\n log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n state_skipped += 1\r\n if state_skipped <= 5:\r\n log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n queued += 1\r\n if queued <= 5 or queued % 5 == 0:\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n size_gb = total_bytes / (1024 ** 3)\r\n log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n log(\r\n \"init\",\r\n f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n )\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n total_scanned = 0\r\n quick_skipped = 0\r\n state_skipped = 0\r\n empty_dirs = 0\r\n queued = 0\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n total_scanned += 1\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n if total_scanned <= 5 or total_scanned % 10 == 0:\r\n log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n quick_skipped += 1\r\n if quick_skipped <= 5:\r\n log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n empty_dirs += 1\r\n if empty_dirs <= 5:\r\n log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n state_skipped += 1\r\n if state_skipped <= 5:\r\n log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n queued += 1\r\n if queued <= 5 or queued % 5 == 0:\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n size_gb = total_bytes / (1024 ** 3)\r\n log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n log(\r\n \"init\",\r\n f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n )\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1733,36 +1733,36 @@ Parameters: ```json { - "relativeWorkspacePath": "zip_sequences.py" + "relativeWorkspacePath": "zip_sequences.py" } ``` Additional data: ```json { - "codeblockId": "69338b80-bad8-4240-beb7-a2e2f95c8c3f" + "codeblockId": "69338b80-bad8-4240-beb7-a2e2f95c8c3f" } ``` Result: ```json { - "diff": { - "chunks": [ + "diff": { + "chunks": [ { - "diffString": " verbose=verbose,\r\n )\r\n+ log(\r\n+ \"scan\",\r\n+ f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n+ f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n+ )\r\n \r\n removed = cleanup_orphan_archives(verbose=verbose)\r", - "oldStart": 870, - "newStart": 870, - "oldLines": 4, - "newLines": 9, - "linesAdded": 5 + "diffString": " verbose=verbose,\r\n )\r\n+ log(\r\n+ \"scan\",\r\n+ f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n+ f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n+ )\r\n \r\n removed = cleanup_orphan_archives(verbose=verbose)\r", + "oldStart": 870, + "newStart": 870, + "oldLines": 4, + "newLines": 9, + "linesAdded": 5 } ], - "editor": "EDITOR_AI" + "editor": "EDITOR_AI" }, - "shouldAutoFixLints": true, - "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", - "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n total_scanned = 0\r\n quick_skipped = 0\r\n state_skipped = 0\r\n empty_dirs = 0\r\n queued = 0\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n total_scanned += 1\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n if total_scanned <= 5 or total_scanned % 10 == 0:\r\n log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n quick_skipped += 1\r\n if quick_skipped <= 5:\r\n log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n empty_dirs += 1\r\n if empty_dirs <= 5:\r\n log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n state_skipped += 1\r\n if state_skipped <= 5:\r\n log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n queued += 1\r\n if queued <= 5 or queued % 5 == 0:\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n size_gb = total_bytes / (1024 ** 3)\r\n log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n log(\r\n \"init\",\r\n f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n )\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" + "shouldAutoFixLints": true, + "resultForModel": "Success. Updated the following files:\nM zip_sequences.py", + "contentsAfterEdit": "#!/usr/bin/env python3\r\n\"\"\"Maintain zipped render sequences for Git hooks.\r\n\r\nDefault mode scans `Renders/`, produces ZIP archives under `Renders/_zipped/`,\r\nand stages any updated archives so commits only track compact files. Switch to\r\n`--mode expand` to inflate the tracked archives back into the ignored working\r\ndirectories after checkouts or pulls.\r\n\"\"\"\r\n\r\nfrom __future__ import annotations\r\n\r\nimport argparse\r\nimport json\r\nimport os\r\nimport platform\r\nimport shutil\r\nimport subprocess\r\nimport sys\r\nimport tempfile\r\nimport time\r\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\r\nfrom pathlib import Path\r\nfrom typing import Iterator, Sequence\r\n\r\n# Try to import psutil for cross-platform RAM detection\r\ntry:\r\n import psutil\r\n HAS_PSUTIL = True\r\nexcept ImportError:\r\n HAS_PSUTIL = False\r\n # For Windows fallback\r\n if platform.system() == \"Windows\":\r\n try:\r\n import ctypes\r\n HAS_CTYPES = True\r\n except ImportError:\r\n HAS_CTYPES = False\r\n else:\r\n HAS_CTYPES = False\r\n\r\n\r\nRENDER_ROOT = Path(\"Renders\")\r\nARCHIVE_ROOT = RENDER_ROOT / \"_zipped\"\r\nSEQUENCE_EXTENSIONS = {\r\n \".png\",\r\n \".jpg\",\r\n \".jpeg\",\r\n \".tif\",\r\n \".tiff\",\r\n \".exr\",\r\n}\r\nSTATE_SUFFIX = \".meta.json\"\r\nDEFAULT_CONFIG = {\r\n \"zipper\": \"7z\",\r\n \"compression\": 9,\r\n \"dailyFormat\": \"daily_YYMMDD\",\r\n \"Max7zInst\": 0, # Maximum concurrent 7z instances (0 = auto-calculate)\r\n}\r\n\r\n\r\ndef log(mode: str, message: str, *, verbose_only: bool = False, verbose: bool = False) -> None:\r\n if verbose_only and not verbose:\r\n return\r\n print(f\"[{mode}] {message}\", flush=True)\r\n\r\n\r\ndef load_config() -> dict:\r\n # First try to load from project's .config folder (current working directory)\r\n # Then fall back to ProjectStructure repo config (next to zip_sequences.py)\r\n cwd = Path.cwd()\r\n project_config = cwd / \".config\" / \"config.json\"\r\n repo_config = Path(__file__).resolve().with_name(\"config.json\")\r\n \r\n config_paths = [\r\n (\"project\", project_config),\r\n (\"repo\", repo_config),\r\n ]\r\n \r\n log(\"init\", \"Loading configuration sources...\")\r\n for source, config_path in config_paths:\r\n try:\r\n if config_path.exists():\r\n log(\"init\", f\"Reading {source} config at {config_path}\")\r\n text = config_path.read_text(encoding=\"utf-8\")\r\n try:\r\n data = json.loads(text)\r\n if isinstance(data, dict):\r\n merged = DEFAULT_CONFIG.copy()\r\n merged.update(data)\r\n log(\"init\", f\"Configuration loaded from {source}\")\r\n return merged\r\n except json.JSONDecodeError:\r\n log(\"init\", f\"Config file at {config_path} is invalid JSON; skipping\")\r\n continue\r\n except OSError:\r\n log(\"init\", f\"Unable to read config at {config_path}; skipping\")\r\n continue\r\n \r\n # If no config found, return defaults\r\n log(\"init\", \"No config files found; using default settings\")\r\n return DEFAULT_CONFIG.copy()\r\n\r\n\r\nCONFIG = load_config()\r\nzipper_val = CONFIG.get(\"zipper\", \"7z\")\r\n# Handle both old boolean format and new string format\r\nif isinstance(zipper_val, bool):\r\n ZIPPER_TYPE = \"7z\" if zipper_val else \"zip\"\r\nelse:\r\n ZIPPER_TYPE = str(zipper_val).lower()\r\n\r\nCOMPRESSION_LEVEL = CONFIG.get(\"compression\", 9)\r\nif isinstance(COMPRESSION_LEVEL, str):\r\n try:\r\n COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)\r\n except ValueError:\r\n COMPRESSION_LEVEL = 9\r\nif not isinstance(COMPRESSION_LEVEL, int):\r\n COMPRESSION_LEVEL = 9\r\nCOMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))\r\n\r\nMAX_7Z_INSTANCES = CONFIG.get(\"Max7zInst\", 0)\r\nif MAX_7Z_INSTANCES is not None:\r\n if isinstance(MAX_7Z_INSTANCES, str):\r\n try:\r\n MAX_7Z_INSTANCES = int(MAX_7Z_INSTANCES)\r\n except ValueError:\r\n MAX_7Z_INSTANCES = 0\r\n if not isinstance(MAX_7Z_INSTANCES, int) or MAX_7Z_INSTANCES < 1:\r\n MAX_7Z_INSTANCES = 0\r\n# Treat 0 as None (auto-calculate)\r\nif MAX_7Z_INSTANCES == 0:\r\n MAX_7Z_INSTANCES = None\r\n\r\nSEVEN_Z_EXE: str | None = None\r\nif ZIPPER_TYPE == \"7z\":\r\n SEVEN_Z_EXE = shutil.which(\"7z\") or shutil.which(\"7za\")\r\n\r\n\r\ndef parse_args() -> argparse.Namespace:\r\n parser = argparse.ArgumentParser(description=\"Sync render sequences with zipped archives.\")\r\n parser.add_argument(\r\n \"--mode\",\r\n choices=(\"zip\", \"expand\"),\r\n default=\"zip\",\r\n help=\"zip sequences for commit (default) or expand tracked archives\",\r\n )\r\n parser.add_argument(\"--jobs\", type=int, help=\"max parallel workers\")\r\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"print extra progress details\")\r\n return parser.parse_args()\r\n\r\n\r\ndef get_available_ram() -> int | None:\r\n \"\"\"Get available RAM in bytes, reserving 20% for system.\r\n \r\n Returns:\r\n Available RAM in bytes, or None if detection fails.\r\n \"\"\"\r\n try:\r\n if HAS_PSUTIL:\r\n # Use psutil for cross-platform RAM detection\r\n mem = psutil.virtual_memory()\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem.total * 0.8)\r\n return available\r\n elif HAS_CTYPES and platform.system() == \"Windows\":\r\n # Windows fallback using ctypes\r\n class MEMORYSTATUSEX(ctypes.Structure):\r\n _fields_ = [\r\n (\"dwLength\", ctypes.c_ulong),\r\n (\"dwMemoryLoad\", ctypes.c_ulong),\r\n (\"ullTotalPhys\", ctypes.c_ulonglong),\r\n (\"ullAvailPhys\", ctypes.c_ulonglong),\r\n (\"ullTotalPageFile\", ctypes.c_ulonglong),\r\n (\"ullAvailPageFile\", ctypes.c_ulonglong),\r\n (\"ullTotalVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailVirtual\", ctypes.c_ulonglong),\r\n (\"ullAvailExtendedVirtual\", ctypes.c_ulonglong),\r\n ]\r\n \r\n kernel32 = ctypes.windll.kernel32\r\n kernel32.GlobalMemoryStatusEx.argtypes = [ctypes.POINTER(MEMORYSTATUSEX)]\r\n kernel32.GlobalMemoryStatusEx.restype = ctypes.c_bool\r\n \r\n mem_status = MEMORYSTATUSEX()\r\n mem_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)\r\n \r\n if kernel32.GlobalMemoryStatusEx(ctypes.byref(mem_status)):\r\n # Reserve 20% for system, use 80% for compression jobs\r\n available = int(mem_status.ullTotalPhys * 0.8)\r\n return available\r\n except Exception:\r\n pass\r\n \r\n return None\r\n\r\n\r\ndef estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int:\r\n \"\"\"Estimate RAM usage per compression job based on folder size.\r\n \r\n Args:\r\n seq_dir: Path to the sequence directory\r\n seq_state: State dictionary containing file information\r\n \r\n Returns:\r\n Estimated RAM usage in bytes\r\n \"\"\"\r\n # Calculate total folder size from seq_state\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n \r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n FIXED_DICT_SIZE_BYTES = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Dictionary RAM: ~11x dictionary size (1GB dict = 11GB RAM for dictionary operations)\r\n # This is the main memory consumer for 7z LZMA compression\r\n dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary\r\n \r\n # Input buffer: 7z processes in chunks, doesn't need full file in RAM\r\n # Use 15% of sequence size, capped at 2GB\r\n input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024)\r\n \r\n # Compression overhead: additional memory for compression process\r\n overhead = 1 * 1024 * 1024 * 1024 # 1GB\r\n \r\n # Total RAM estimate\r\n estimated_ram = dict_ram + input_buffer + overhead\r\n \r\n return estimated_ram\r\n else:\r\n # zip compression is more memory-efficient\r\n # Conservative estimate: 1GB per job\r\n return 1024 * 1024 * 1024 # 1 GB\r\n\r\n\r\ndef max_workers(\r\n requested: int | None,\r\n work_items: list[tuple[Path, Path, Path, dict]] | None = None,\r\n *,\r\n verbose: bool = False\r\n) -> tuple[int, int | None]:\r\n \"\"\"Calculate maximum worker count based on CPU and RAM constraints.\r\n \r\n Args:\r\n requested: User-requested worker count (from --jobs)\r\n work_items: List of work items (seq_dir, zip_path, state_path, seq_state)\r\n verbose: Whether to log RAM-based calculations\r\n \r\n Returns:\r\n Tuple of (worker_count, per_job_memory_limit_bytes)\r\n per_job_memory_limit_bytes is None if not using 7z or RAM detection failed\r\n \"\"\"\r\n cpu = os.cpu_count() or 1\r\n cpu_limit = max(1, min(8, cpu))\r\n if requested and requested > 0:\r\n cpu_limit = min(requested, max(1, cpu))\r\n \r\n # If no work items provided, return CPU-based limit\r\n if work_items is None or len(work_items) == 0:\r\n return (cpu_limit, {})\r\n \r\n # Try to calculate RAM-based limit\r\n available_ram = get_available_ram()\r\n \r\n if available_ram is None:\r\n # RAM detection failed, fall back to CPU limit\r\n if verbose:\r\n log(\"zip\", \"RAM detection failed, using CPU-based worker limit\", verbose_only=True, verbose=verbose)\r\n return (cpu_limit, {})\r\n \r\n # For 7z: use fixed dictionary size and calculate workers\r\n if ZIPPER_TYPE == \"7z\":\r\n # Fixed dictionary size: 1GB (1024MB)\r\n FIXED_DICT_SIZE_MB = 1024\r\n fixed_dict_size_bytes = FIXED_DICT_SIZE_MB * 1024 * 1024\r\n \r\n # Check if Max7zInst is configured\r\n if MAX_7Z_INSTANCES is not None:\r\n # Use configured maximum instances, but still respect user's --jobs and work items\r\n final_limit = MAX_7Z_INSTANCES\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n final_limit = min(final_limit, num_work_items)\r\n if requested and requested > 0:\r\n final_limit = min(final_limit, requested)\r\n \r\n # Create RAM limits dict (all use fixed dict size, but return as dict for consistency)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = fixed_dict_size_bytes\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"Using Max7zInst={MAX_7Z_INSTANCES} from config → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n \r\n # Auto-calculate based on RAM if Max7zInst not configured\r\n # Use bin-packing algorithm with size-aware RAM estimation\r\n if available_ram is not None:\r\n # available_ram is already 80% of total (20% reserved for system)\r\n # Use 95% of available RAM for compression jobs\r\n compression_ram = int(available_ram * 0.95)\r\n \r\n # Estimate RAM for each work item and create list with (seq_dir, estimated_ram, work_item)\r\n work_items_with_ram: list[tuple[Path, int, tuple[Path, Path, Path, dict]]] = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n \r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n work_items_with_ram.append((seq_dir, estimated_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use a safe default (12GB minimum for 1GB dict)\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB\r\n work_items_with_ram.append((seq_dir, default_ram, (seq_dir, zip_path, state_path, seq_state)))\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n # Sort by estimated RAM (largest first) for bin-packing\r\n work_items_with_ram.sort(key=lambda x: x[1], reverse=True)\r\n \r\n # Bin-packing algorithm: pack largest items first\r\n bins: list[list[tuple[Path, int, tuple[Path, Path, Path, dict]]]] = []\r\n bin_remaining: list[int] = []\r\n \r\n for seq_dir, estimated_ram, work_item in work_items_with_ram:\r\n # Try to fit in existing bin\r\n placed = False\r\n for i, remaining in enumerate(bin_remaining):\r\n if remaining >= estimated_ram:\r\n bins[i].append((seq_dir, estimated_ram, work_item))\r\n bin_remaining[i] -= estimated_ram\r\n placed = True\r\n break\r\n \r\n # If doesn't fit, create new bin\r\n if not placed:\r\n bins.append([(seq_dir, estimated_ram, work_item)])\r\n bin_remaining.append(compression_ram - estimated_ram)\r\n \r\n # Worker count is number of bins\r\n worker_count = len(bins)\r\n \r\n # Cap at number of actual work items (can't have more workers than jobs)\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n worker_count = min(worker_count, num_work_items)\r\n \r\n # Respect user's --jobs if provided\r\n if requested and requested > 0:\r\n worker_count = min(worker_count, requested)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n compression_ram_gb = compression_ram / (1024 ** 3)\r\n total_estimated_gb = sum(ram for _, ram, _ in work_items_with_ram) / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%)\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Estimated RAM per sequence: {total_estimated_gb:.1f}GB total across {num_work_items} sequences\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n if len(bins) > 0:\r\n bin_sizes = [sum(ram for _, ram, _ in bin_items) / (1024 ** 3) for bin_items in bins]\r\n log(\r\n \"zip\",\r\n f\"Bin-packing: {worker_count} workers, bin sizes: {[f'{s:.1f}GB' for s in bin_sizes[:5]]}{'...' if len(bin_sizes) > 5 else ''}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n log(\r\n \"zip\",\r\n f\"Final worker count: {worker_count} (requested: {requested})\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n # Return worker count and RAM limits dict\r\n return (worker_count, ram_limits_dict)\r\n \r\n # RAM detection failed, use a safe default (no CPU limit)\r\n default_limit = 4\r\n num_work_items = len(work_items) if work_items else 0\r\n if num_work_items > 0:\r\n default_limit = min(default_limit, num_work_items)\r\n if requested and requested > 0:\r\n default_limit = min(default_limit, requested)\r\n \r\n # Create RAM limits dict with safe defaults (12GB per job for 1GB dict)\r\n ram_limits_dict: dict[Path, int] = {}\r\n if work_items:\r\n default_ram = 12 * 1024 * 1024 * 1024 # 12GB default\r\n for seq_dir, _, _, _ in work_items:\r\n ram_limits_dict[seq_dir] = default_ram\r\n \r\n if verbose:\r\n log(\r\n \"zip\",\r\n f\"RAM detection failed and Max7zInst not set, using default worker limit → \"\r\n f\"work items: {num_work_items}, requested: {requested}, final: {default_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n return (default_limit, ram_limits_dict)\r\n \r\n # For zip compression, use existing estimation-based approach\r\n # Estimate RAM per job for each work item\r\n ram_estimates = []\r\n ram_limits_dict: dict[Path, int] = {}\r\n for seq_dir, zip_path, state_path, seq_state in work_items:\r\n try:\r\n estimated_ram = estimate_ram_per_job(seq_dir, seq_state)\r\n ram_estimates.append(estimated_ram)\r\n ram_limits_dict[seq_dir] = estimated_ram\r\n except Exception:\r\n # If estimation fails, use fallback estimate\r\n fallback_ram = 1024 * 1024 * 1024 # 1GB fallback for zip\r\n ram_estimates.append(fallback_ram)\r\n ram_limits_dict[seq_dir] = fallback_ram\r\n \r\n if not ram_estimates:\r\n return (cpu_limit, {})\r\n \r\n max_ram_per_job = max(ram_estimates)\r\n ram_limit = max(1, available_ram // max_ram_per_job)\r\n ram_limit = min(ram_limit, 6) # Conservative limit for zip\r\n final_limit = min(cpu_limit, ram_limit)\r\n \r\n if verbose:\r\n ram_gb = available_ram / (1024 ** 3)\r\n max_ram_gb = max_ram_per_job / (1024 ** 3)\r\n log(\r\n \"zip\",\r\n f\"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → \"\r\n f\"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}\",\r\n verbose_only=True,\r\n verbose=verbose\r\n )\r\n \r\n return (final_limit, ram_limits_dict)\r\n\r\n\r\ndef is_archive_path(path: Path) -> bool:\r\n return any(part in (\"_archive\", \"_CURRENT\") for part in path.parts)\r\n\r\n\r\ndef find_sequence_dirs(root: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(root):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames)\r\n if has_frames:\r\n yield path\r\n\r\n\r\ndef iter_sequence_files(seq_dir: Path) -> Iterator[Path]:\r\n for dirpath, dirnames, filenames in os.walk(seq_dir):\r\n path = Path(dirpath)\r\n dirnames[:] = [d for d in dirnames if d not in (\"_archive\", \"_CURRENT\")]\r\n if is_archive_path(path):\r\n continue\r\n for filename in filenames:\r\n yield path / filename\r\n\r\n\r\ndef compute_state(seq_dir: Path) -> dict:\r\n entries = []\r\n files = sorted(\r\n iter_sequence_files(seq_dir),\r\n key=lambda p: p.relative_to(seq_dir).as_posix(),\r\n )\r\n for file_path in files:\r\n stat = file_path.stat()\r\n entries.append(\r\n {\r\n \"path\": file_path.relative_to(seq_dir).as_posix(),\r\n \"size\": stat.st_size,\r\n \"mtime_ns\": stat.st_mtime_ns,\r\n }\r\n )\r\n return {\"files\": entries}\r\n\r\n\r\ndef current_state(seq_dir: Path) -> dict:\r\n if not seq_dir.exists() or not seq_dir.is_dir():\r\n return {\"files\": []}\r\n return compute_state(seq_dir)\r\n\r\n\r\ndef load_state(state_path: Path) -> dict | None:\r\n if not state_path.exists():\r\n return None\r\n try:\r\n return json.loads(state_path.read_text())\r\n except json.JSONDecodeError:\r\n return None\r\n\r\n\r\ndef state_changed(seq_state: dict, stored_state: dict | None) -> bool:\r\n if stored_state is None:\r\n return True\r\n return seq_state != stored_state\r\n\r\n\r\ndef archive_path_for(seq_dir: Path) -> Path:\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n suffix = \".7z\" if ZIPPER_TYPE == \"7z\" else \".zip\"\r\n return (ARCHIVE_ROOT / rel).with_suffix(suffix)\r\n\r\n\r\ndef sequence_dir_for(zip_path: Path) -> Path:\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n return (RENDER_ROOT / rel).with_suffix(\"\")\r\n\r\n\r\ndef state_path_for(zip_path: Path) -> Path:\r\n return zip_path.with_suffix(zip_path.suffix + STATE_SUFFIX)\r\n\r\n\r\ndef zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None:\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z compression requested but 7z executable not found in PATH. \"\r\n \"Please install 7z (e.g., via Chocolatey: choco install 7zip) \"\r\n \"or set zipper to 'zip' in config.json\"\r\n )\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n \r\n # If creating a .7z file, remove any existing .zip file for the same sequence\r\n if zip_path.suffix == \".7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n \r\n # Build list of files to archive with relative paths\r\n file_list = []\r\n for file_path in iter_sequence_files(seq_dir):\r\n rel_path = file_path.relative_to(seq_dir).as_posix()\r\n file_list.append(rel_path)\r\n \r\n if not file_list:\r\n raise RuntimeError(f\"No files found to archive in {seq_dir}\")\r\n \r\n # Create zip in temporary location first to avoid issues with corrupted existing files\r\n temp_zip = None\r\n list_file_path = None\r\n try:\r\n # Create temporary archive file path (but don't create the file - let 7z create it)\r\n temp_zip_path = tempfile.mktemp(suffix=\".7z\", dir=zip_path.parent)\r\n temp_zip = Path(temp_zip_path)\r\n \r\n # Create list file with absolute path\r\n fd, temp_path = tempfile.mkstemp(suffix=\".lst\", text=True)\r\n list_file_path = Path(temp_path)\r\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as list_file:\r\n for rel_path in file_list:\r\n list_file.write(rel_path + \"\\n\")\r\n list_file.flush()\r\n os.fsync(list_file.fileno()) # Ensure data is written to disk\r\n # File is closed here by context manager, small delay to ensure OS releases handle\r\n time.sleep(0.1)\r\n \r\n # Use absolute paths for both list file and temp zip\r\n list_file_abs = list_file_path.resolve()\r\n temp_zip_abs = temp_zip.resolve()\r\n # Create archive in temp location first (7z will create it fresh)\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"a\",\r\n \"-y\",\r\n \"-bb0\", # Suppress progress output\r\n f\"-mx={COMPRESSION_LEVEL}\",\r\n \"-t7z\", # Use 7z format, not zip\r\n ]\r\n \r\n # Always use fixed dictionary size: 1GB (1024MB)\r\n # The per_job_memory_limit parameter is the estimated RAM usage (for logging/info only)\r\n # We keep dictionary at 1GB for best compression regardless of RAM estimate\r\n FIXED_DICT_SIZE_MB = 1024\r\n cmd.append(f\"-md={FIXED_DICT_SIZE_MB}m\")\r\n \r\n cmd.extend([\r\n str(temp_zip_abs),\r\n f\"@{list_file_abs}\",\r\n ])\r\n result = subprocess.run(\r\n cmd,\r\n cwd=seq_dir,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z compression failed: {error_msg}\")\r\n \r\n # Move temp zip to final location, replacing any existing file\r\n if zip_path.exists():\r\n zip_path.unlink()\r\n temp_zip.replace(zip_path)\r\n temp_zip = None # Mark as moved so we don't delete it\r\n finally:\r\n # Clean up temp zip if it wasn't moved\r\n if temp_zip and temp_zip.exists():\r\n try:\r\n temp_zip.unlink(missing_ok=True)\r\n except OSError:\r\n pass\r\n # Clean up list file, with retry in case 7z still has it open\r\n if list_file_path and list_file_path.exists():\r\n for attempt in range(3):\r\n try:\r\n list_file_path.unlink(missing_ok=True)\r\n break\r\n except PermissionError:\r\n if attempt < 2:\r\n time.sleep(0.1) # Wait 100ms before retry\r\n else:\r\n # Last attempt failed, just log and continue\r\n # The temp file will be cleaned up by the OS eventually\r\n pass\r\n return\r\n\r\n # Use zipfile (only if ZIPPER_TYPE == \"zip\")\r\n if ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile\r\n\r\n zip_path.parent.mkdir(parents=True, exist_ok=True)\r\n if COMPRESSION_LEVEL <= 0:\r\n compression = ZIP_STORED\r\n zip_kwargs = {}\r\n else:\r\n compression = ZIP_DEFLATED\r\n zip_kwargs = {\"compresslevel\": COMPRESSION_LEVEL}\r\n\r\n with ZipFile(zip_path, \"w\", compression=compression, **zip_kwargs) as archive:\r\n for file_path in iter_sequence_files(seq_dir):\r\n archive.write(file_path, arcname=file_path.relative_to(seq_dir).as_posix())\r\n return\r\n\r\n # Unknown ZIPPER_TYPE - fail with clear error\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n\r\ndef expand_sequence(zip_path: Path, seq_state: dict) -> None:\r\n target_dir = sequence_dir_for(zip_path)\r\n if target_dir.exists():\r\n shutil.rmtree(target_dir)\r\n target_dir.mkdir(parents=True, exist_ok=True)\r\n\r\n if ZIPPER_TYPE == \"7z\":\r\n if SEVEN_Z_EXE is None:\r\n raise RuntimeError(\r\n \"7z extraction requested but 7z executable not found in PATH. \"\r\n \"Please install 7z or set zipper to 'zip' in config.json\"\r\n )\r\n cmd = [\r\n SEVEN_Z_EXE,\r\n \"x\",\r\n \"-y\",\r\n str(zip_path),\r\n f\"-o{target_dir}\",\r\n ]\r\n result = subprocess.run(\r\n cmd,\r\n check=False,\r\n stdout=subprocess.PIPE,\r\n stderr=subprocess.PIPE,\r\n text=True,\r\n )\r\n if result.returncode != 0:\r\n error_msg = result.stderr.strip() if result.stderr else \"Unknown error\"\r\n if result.stdout:\r\n error_msg += f\"\\nstdout: {result.stdout.strip()}\"\r\n raise RuntimeError(f\"7z extraction failed: {error_msg}\")\r\n elif ZIPPER_TYPE == \"zip\":\r\n from zipfile import ZipFile\r\n\r\n with ZipFile(zip_path, \"r\") as archive:\r\n archive.extractall(target_dir)\r\n else:\r\n raise RuntimeError(\r\n f\"Unsupported ZIPPER_TYPE: {ZIPPER_TYPE!r}. \"\r\n f\"Expected '7z' or 'zip'. \"\r\n f\"Config zipper value: {CONFIG.get('zipper', 'not set')!r}\"\r\n )\r\n\r\n for entry in seq_state.get(\"files\", []):\r\n file_path = target_dir / entry[\"path\"]\r\n if file_path.exists():\r\n os.utime(file_path, ns=(entry[\"mtime_ns\"], entry[\"mtime_ns\"]))\r\n\r\n\r\ndef process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]:\r\n log(\"zip\", f\"{seq_dir} -> {zip_path}\", verbose_only=True, verbose=verbose)\r\n zip_sequence(seq_dir, zip_path, per_job_memory_limit)\r\n state_path.write_text(json.dumps(seq_state, indent=2))\r\n return (zip_path, state_path)\r\n\r\n\r\ndef process_expand(zip_path: Path, state: dict, *, verbose: bool) -> None:\r\n log(\"expand\", f\"{zip_path} -> {sequence_dir_for(zip_path)}\", verbose_only=True, verbose=verbose)\r\n expand_sequence(zip_path, state)\r\n\r\n\r\ndef run_zip(requested_workers: int | None, *, verbose: bool) -> int:\r\n work_items: list[tuple[Path, Path, Path, dict]] = []\r\n log(\"init\", f\"Scanning sequences under {RENDER_ROOT.resolve()}\")\r\n total_scanned = 0\r\n quick_skipped = 0\r\n state_skipped = 0\r\n empty_dirs = 0\r\n queued = 0\r\n\r\n if RENDER_ROOT.exists():\r\n for seq_dir in find_sequence_dirs(RENDER_ROOT):\r\n total_scanned += 1\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n if total_scanned <= 5 or total_scanned % 10 == 0:\r\n log(\"scan\", f\"[{total_scanned}] Inspecting {rel}\")\r\n\r\n # Get the target archive path (will be .7z if ZIPPER_TYPE is \"7z\")\r\n zip_path = archive_path_for(seq_dir)\r\n state_path = state_path_for(zip_path)\r\n \r\n # Quick check: if archive exists, load stored state first (fast)\r\n stored_state = load_state(state_path)\r\n \r\n # Check if we need to upgrade from .zip to .7z\r\n old_zip_path = None\r\n old_stored_state = None\r\n if ZIPPER_TYPE == \"7z\":\r\n old_zip_path = zip_path.with_suffix(\".zip\")\r\n if old_zip_path.exists():\r\n old_state_path = state_path_for(old_zip_path)\r\n old_stored_state = load_state(old_state_path)\r\n # If old .zip exists and .7z doesn't, use old .zip's state for comparison\r\n if not zip_path.exists() and old_stored_state is not None:\r\n stored_state = old_stored_state\r\n \r\n # If .7z archive exists and we have stored state, do quick check before computing full state\r\n if zip_path.exists() and stored_state is not None:\r\n # Quick check: if directory mtime is older than archive, likely unchanged\r\n try:\r\n dir_mtime = seq_dir.stat().st_mtime_ns\r\n archive_mtime = zip_path.stat().st_mtime_ns\r\n # If directory wasn't modified since archive was created, skip state computation\r\n if dir_mtime <= archive_mtime:\r\n quick_skipped += 1\r\n if quick_skipped <= 5:\r\n log(\"scan\", f\"Skipping {rel} (unchanged since archive)\")\r\n # Still need to check for old .zip cleanup (we have .7z, so .zip is obsolete)\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n continue\r\n except OSError:\r\n # If stat fails, fall through to full state computation\r\n pass\r\n \r\n # Compute current state only if we need to\r\n seq_state = compute_state(seq_dir)\r\n if not seq_state[\"files\"]:\r\n empty_dirs += 1\r\n if empty_dirs <= 5:\r\n log(\"scan\", f\"{rel} has no files; skipping\")\r\n continue\r\n \r\n # Check if state changed\r\n if stored_state is not None and not state_changed(seq_state, stored_state):\r\n # Metadata matches stored state\r\n state_skipped += 1\r\n if state_skipped <= 5:\r\n log(\"scan\", f\"{rel} metadata unchanged; archive up to date\")\r\n if zip_path.exists():\r\n # .7z exists and is up to date, clean up old .zip if it exists\r\n if old_zip_path and old_zip_path.exists():\r\n old_zip_path.unlink(missing_ok=True)\r\n old_state_path = state_path_for(old_zip_path)\r\n if old_state_path.exists():\r\n old_state_path.unlink(missing_ok=True)\r\n elif old_zip_path and old_zip_path.exists() and old_stored_state is not None:\r\n # .7z doesn't exist, but .zip exists and metadata matches\r\n # Keep the .zip file, don't create .7z\r\n continue\r\n else:\r\n # No archive exists, but state matches (shouldn't happen, but be safe)\r\n continue\r\n\r\n work_items.append((seq_dir, zip_path, state_path, seq_state))\r\n queued += 1\r\n if queued <= 5 or queued % 5 == 0:\r\n total_bytes = sum(entry.get(\"size\", 0) for entry in seq_state.get(\"files\", []))\r\n size_gb = total_bytes / (1024 ** 3)\r\n log(\"scan\", f\"Queued {rel} for compression (~{size_gb:.2f}GB) [{queued} total]\")\r\n\r\n if not work_items:\r\n if not RENDER_ROOT.exists():\r\n log(\"zip\", \"Render root 'Renders' not found; nothing to zip.\")\r\n else:\r\n log(\"zip\", \"Archives already up to date; no sequences needed zipping.\")\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n return 0\r\n\r\n # Calculate RAM-aware worker count based on work items\r\n worker_count, ram_limits_dict = max_workers(requested_workers, work_items, verbose=verbose)\r\n log(\r\n \"init\",\r\n f\"Preparing to compress {len(work_items)} sequence(s) with {worker_count} worker(s)\",\r\n )\r\n\r\n updated_paths: list[Path] = []\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_zip, seq_dir, zip_path, state_path, seq_state, ram_limits_dict.get(seq_dir), verbose=verbose): seq_dir\r\n for seq_dir, zip_path, state_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n updated_paths.extend(future.result())\r\n completed += 1\r\n seq_dir = future_map[future]\r\n rel = seq_dir.relative_to(RENDER_ROOT)\r\n log(\"zip\", f\"{completed}/{total} {rel}\")\r\n\r\n updated_count = len(updated_paths) // 2\r\n log(\"zip\", f\"Updated {updated_count} sequence archive(s).\", verbose=verbose)\r\n if updated_paths:\r\n log(\r\n \"zip\",\r\n \"Archives updated. Stage manually with `git add Renders/_zipped`, if desired.\",\r\n verbose_only=True,\r\n verbose=verbose,\r\n )\r\n log(\r\n \"scan\",\r\n f\"Summary: scanned {total_scanned}, quick-skipped {quick_skipped}, \"\r\n f\"state-skipped {state_skipped}, empty {empty_dirs}, queued {queued}\",\r\n )\r\n\r\n removed = cleanup_orphan_archives(verbose=verbose)\r\n if removed:\r\n log(\"zip\", f\"Removed {removed} orphan archive(s).\", verbose=verbose)\r\n\r\n return updated_count\r\n\r\n\r\ndef run_expand(worker_count: int, *, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n log(\"expand\", \"No archives to expand (missing 'Renders/_zipped').\")\r\n return 0\r\n\r\n work_items: list[tuple[Path, dict]] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n state_path = state_path_for(zip_path)\r\n seq_state = load_state(state_path)\r\n if seq_state is None:\r\n log(\"expand\", f\"Skipping {zip_path} (missing metadata)\")\r\n continue\r\n\r\n target_dir = sequence_dir_for(zip_path)\r\n if current_state(target_dir) == seq_state:\r\n continue\r\n\r\n work_items.append((zip_path, seq_state))\r\n\r\n if not work_items:\r\n log(\"expand\", \"Working folders already match archives; nothing to expand.\")\r\n return 0\r\n\r\n total = len(work_items)\r\n completed = 0\r\n\r\n with ThreadPoolExecutor(max_workers=worker_count) as executor:\r\n future_map = {\r\n executor.submit(process_expand, zip_path, seq_state, verbose=verbose): zip_path\r\n for zip_path, seq_state in work_items\r\n }\r\n\r\n for future in as_completed(future_map):\r\n future.result()\r\n completed += 1\r\n zip_path = future_map[future]\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"expand\", f\"{completed}/{total} {rel}\")\r\n\r\n log(\"expand\", f\"Refreshed {len(work_items)} sequence folder(s).\", verbose=verbose)\r\n return len(work_items)\r\n\r\n\r\ndef cleanup_orphan_archives(*, verbose: bool) -> int:\r\n if not ARCHIVE_ROOT.exists():\r\n return 0\r\n\r\n removed: list[Path] = []\r\n\r\n # Look for both .zip and .7z archives\r\n archive_patterns = [\"*.zip\", \"*.7z\"]\r\n for pattern in archive_patterns:\r\n for zip_path in ARCHIVE_ROOT.rglob(pattern):\r\n seq_dir = sequence_dir_for(zip_path)\r\n if seq_dir.exists():\r\n continue\r\n\r\n rel = zip_path.relative_to(ARCHIVE_ROOT)\r\n log(\"zip\", f\"Removing orphan archive {rel}\", verbose_only=True, verbose=verbose)\r\n\r\n zip_path.unlink(missing_ok=True)\r\n state_path = state_path_for(zip_path)\r\n if state_path.exists():\r\n state_path.unlink()\r\n removed.append(zip_path)\r\n\r\n if not removed:\r\n return 0\r\n\r\n for parent in sorted({p.parent for p in removed}, key=lambda p: len(p.parts), reverse=True):\r\n if not parent.exists():\r\n continue\r\n while parent != ARCHIVE_ROOT and not any(parent.iterdir()):\r\n parent.rmdir()\r\n parent = parent.parent\r\n\r\n return len(removed)\r\n\r\n\r\ndef main() -> int:\r\n args = parse_args()\r\n log(\"init\", \"zip_sequences starting up...\")\r\n log(\"init\", f\"Working directory: {Path.cwd()}\")\r\n log(\"init\", f\"Mode: {args.mode}, zipper: {ZIPPER_TYPE}, jobs arg: {args.jobs or 'auto'}\")\r\n if ZIPPER_TYPE == \"7z\":\r\n exe = SEVEN_Z_EXE or \"not found\"\r\n max_inst = MAX_7Z_INSTANCES if MAX_7Z_INSTANCES is not None else \"auto\"\r\n log(\"init\", f\"7z executable: {exe}, Max7zInst: {max_inst}\")\r\n\r\n if args.mode == \"expand\":\r\n # For expand mode, use simple CPU-based worker calculation\r\n workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose)\r\n run_expand(workers, verbose=args.verbose)\r\n return 0\r\n\r\n # For zip mode, work items will be calculated in run_zip\r\n updated = run_zip(args.jobs, verbose=args.verbose)\r\n return 0 if updated >= 0 else 1\r\n\r\n\r\nif __name__ == \"__main__\":\r\n try:\r\n raise SystemExit(main())\r\n except Exception as exc: # broad to surface unexpected errors cleanly\r\n print(f\"Sequence sync failed: {exc}\", file=sys.stderr)\r\n raise\r\n\r\n" } ``` User decision: **accepted** @@ -1986,7 +1986,7 @@ Logging looks good. RAM estimation is too low for very large sequences (e.g., 66 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 14, lines removed: 4 @@ -1995,7 +1995,7 @@ Lines added: 14, lines removed: 4 @@ -218,10 +218,20 @@ dict_ram = FIXED_DICT_SIZE_BYTES * 11 # 11GB for 1GB dictionary -- # Input buffer: 7z processes in chunks, doesn't need full file in RAM +- # Input buffer: 7z processes in chunks, doesn't need full file in RAM - # Use 15% of sequence size, capped at 2GB - input_buffer = min(int(total_bytes * 0.15), 2 * 1024 * 1024 * 1024) + # Input buffer: 7z processes in chunks, but for very large sequences needs more RAM @@ -2087,7 +2087,7 @@ For a 668GB sequence with 14,402 files, 7z needs more RAM for metadata and direc -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 7, lines removed: 8 diff --git a/.specstory/history/2025-12-17_07-34Z-configloader-structdir-path-issue.md b/.specstory/history/2025-12-17_07-34Z-configloader-structdir-path-issue.md new file mode 100644 index 0000000..6fb9041 --- /dev/null +++ b/.specstory/history/2025-12-17_07-34Z-configloader-structdir-path-issue.md @@ -0,0 +1,184 @@ + + + + +# ConfigLoader structDir path issue (2025-12-17 07:34Z) + +_**User**_ + +This should be capable of running anywhere and turning the structDir into wherever the script was run from. + +PS R:\Creative\artsy\maya\0 ProjectStructure> .\ConfigLoader.ps1 +Paste the project path to deploy in: "C:\Users\Nathan\Desktop\2025-12-17 Little Reindeer" +Write-Error: Configured structDir not found: A:\1 Amazon_Active_Projects\3 ProjectStructure +PS R:\Creative\artsy\maya\0 ProjectStructure> + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +Examining `ConfigLoader.ps1` to understand its current behavior. + + +--- + + +
Tool use: **read_file_v2** • Read file: R:\Creative\artsy\maya\0 ProjectStructure\ConfigLoader.ps1 + +
+ + +--- + + +
Tool use: **read_file_v2** • Read file: R:\Creative\artsy\maya\0 ProjectStructure\config.json + +
+ + +--- + + + +--- + + +The script reads `structDir` from `config.json`, which has a hardcoded path that doesn't exist. Updating it to always use the script's directory as `structDir` when run directly, regardless of the config. + + +--- + + +
Tool use: **read_file_v2** • Read file: R:\Creative\artsy\maya\0 ProjectStructure\ConfigLoader.ps1 + +
+ + +--- + + + +--- + + +Updating `Get-StructDirectory` to always use the script's directory when run directly, ignoring the config value for portability. + + +--- + + +
+Tool use: **edit_file_v2** + +Additional data: + +```json +{ + "codeblockId": "10c3c2bd-ef53-4f35-ba2d-93ee7c232817" +} +``` +Status: **completed** + + +
+ + +--- + + + +--- + + +Updated `Get-StructDirectory` to always use the script's directory when run directly, ignoring the config value. This makes the script portable—you can run it from any location and it will use its own directory as `structDir`. + +When dot-sourced (imported by other scripts), it still respects the config value for backward compatibility. + +Checking for linter errors: + + +--- + + +
+Tool use: **read_lints** + + +Parameters: + +```json +{ + "paths": [ + "R:\\Creative\\artsy\\maya\\0 ProjectStructure\\ConfigLoader.ps1" + ] +} +``` +Result: + +```json +{ + "linterErrorsByFile": [ + { + "relativeWorkspacePath": "R:\\Creative\\artsy\\maya\\0 ProjectStructure\\ConfigLoader.ps1" + } + ] +} +``` +Status: **completed** + + +
+ + +--- + + + +--- + + +Change complete. The script now uses its own directory as `structDir` when run directly, regardless of the config value. You can run it from any location and it will work correctly. + +--- + +_**User**_ + +Let's have it change the structDir value in config.json + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +Updating the script to set `structDir` in `config.json` to the script's directory when run directly. + + +--- + + +
Tool use: **read_file_v2** • Read file: R:\Creative\artsy\maya\0 ProjectStructure\ConfigLoader.ps1 + +
+ + +--- + + + +--- + + +Adding a function to update `config.json` and calling it when the script runs directly: + + +--- + diff --git a/ConfigLoader.ps1 b/ConfigLoader.ps1 index bebdd79..92e6d3a 100644 --- a/ConfigLoader.ps1 +++ b/ConfigLoader.ps1 @@ -45,6 +45,11 @@ function Get-ConfigValue { } function Get-StructDirectory { + # When script is run directly (not dot-sourced), always use script's directory for portability + if ($MyInvocation.InvocationName -ne '.') { + return $script:LoaderRoot + } + $value = Get-ConfigValue -Name 'structDir' if ($null -eq $value -or [string]::IsNullOrWhiteSpace($value)) { return $script:LoaderRoot @@ -181,9 +186,30 @@ function Get-ZipCompressionLevel { return [Math]::Min(9, [Math]::Max(0, $value)) } +function Update-ConfigStructDir { + param( + [Parameter(Mandatory)] [string]$StructDir + ) + + $config = Get-ProjectStructureConfig + $config.structDir = $StructDir + + try { + $json = $config | ConvertTo-Json -Depth 10 + Set-Content -LiteralPath $script:ConfigPath -Value $json -NoNewline -ErrorAction Stop + $script:ConfigCache = $config + } + catch { + Write-Warning "Failed to update config.json: $($_.Exception.Message)" + } +} + # If script is run directly (not dot-sourced), prompt for project path and deploy # When dot-sourced, InvocationName is '.'; when run directly, it's the script path or name if ($MyInvocation.InvocationName -ne '.') { + # Update config.json to use script's directory as structDir for portability + Update-ConfigStructDir -StructDir $script:LoaderRoot + $projectPath = Get-ProjectPathFromUser if ($null -ne $projectPath) { # Deploy batch files and config to the project diff --git a/config.json b/config.json index 20eb7fd..71c04d8 100644 --- a/config.json +++ b/config.json @@ -1,8 +1,7 @@ { - "dailyFormat": "daily_YYMMDD", - "structDir": "A:\\1 Amazon_Active_Projects\\3 ProjectStructure", + "dailyFormat": "YYYY-MM-DD", + "structDir": "R:\\Creative\\artsy\\maya\\0 ProjectStructure", "zipper": "7z", "compression": 0, "Max7zInst": 0 -} - +} \ No newline at end of file