From 45dc5956a2ecb643808590ee6e7b66489c90dd53 Mon Sep 17 00:00:00 2001 From: Raincloud Date: Fri, 26 Dec 2025 18:12:53 -0700 Subject: [PATCH] ignore RnR universally --- ...o-overwrite-gitattributes-and-gitignore.md | 543 +++++++++--------- components/gitignore | 2 +- config.json | 2 +- 3 files changed, 273 insertions(+), 274 deletions(-) diff --git a/.specstory/history/2025-11-11_01-50Z-update-configloader-to-overwrite-gitattributes-and-gitignore.md b/.specstory/history/2025-11-11_01-50Z-update-configloader-to-overwrite-gitattributes-and-gitignore.md index f6b443b..5ce26d7 100644 --- a/.specstory/history/2025-11-11_01-50Z-update-configloader-to-overwrite-gitattributes-and-gitignore.md +++ b/.specstory/history/2025-11-11_01-50Z-update-configloader-to-overwrite-gitattributes-and-gitignore.md @@ -58,19 +58,19 @@ Updating `ConfigLoader.ps1` to include `gitattributes` and `gitignore` in the de -
Tool use: **code_edit** • Edit file: ConfigLoader.ps1 +
Tool use: **search_replace** • Edit file: ConfigLoader.ps1 **Chunk 1** Lines added: 3, lines removed: 1 ```diff @@ -215,5 +215,7 @@ - @{ Name = 'ZipSeqArchv.bat'; Source = Join-Path -Path $structDir -ChildPath 'ZipSeqArchv.bat' }, - @{ Name = 'UnzipSeqArchv.bat'; Source = Join-Path -Path $structDir -ChildPath 'UnzipSeqArchv.bat' }, -- @{ Name = 'NewDaily.bat'; Source = Join-Path -Path $structDir -ChildPath 'NewDaily.bat' } -+ @{ Name = 'NewDaily.bat'; Source = Join-Path -Path $structDir -ChildPath 'NewDaily.bat' }, -+ @{ Name = '.gitattributes'; Source = Join-Path -Path $structDir -ChildPath 'components\gitattributes' }, -+ @{ Name = '.gitignore'; Source = Join-Path -Path $structDir -ChildPath 'components\gitignore' } + @{ Name = 'ZipSeqArchv.bat'; Source = Join-Path -Path $structDir -ChildPath 'ZipSeqArchv.bat' }, + @{ Name = 'UnzipSeqArchv.bat'; Source = Join-Path -Path $structDir -ChildPath 'UnzipSeqArchv.bat' }, +- @{ Name = 'NewDaily.bat'; Source = Join-Path -Path $structDir -ChildPath 'NewDaily.bat' } ++ @{ Name = 'NewDaily.bat'; Source = Join-Path -Path $structDir -ChildPath 'NewDaily.bat' }, ++ @{ Name = '.gitattributes'; Source = Join-Path -Path $structDir -ChildPath 'components\gitattributes' }, ++ @{ Name = '.gitignore'; Source = Join-Path -Path $structDir -ChildPath 'components\gitignore' } ) ``` @@ -261,7 +261,7 @@ Adding the Substance file extensions to the source `gitattributes` file: -
Tool use: **code_edit** • Edit file: components/gitattributes +
Tool use: **search_replace** • Edit file: components/gitattributes **Chunk 1** Lines added: 4, lines removed: 2 @@ -331,7 +331,7 @@ Updating the script to also deploy `.gitattributes` and `.gitignore` to the proj -
Tool use: **code_edit** • Edit file: ConfigLoader.ps1 +
Tool use: **search_replace** • Edit file: ConfigLoader.ps1 **Chunk 1** Lines added: 19, lines removed: undefined @@ -339,23 +339,23 @@ Lines added: 19, lines removed: undefined ```diff @@ -312,4 +312,23 @@ - Write-Host "All batch files refreshed successfully." -ForegroundColor Green + Write-Host "All batch files refreshed successfully." -ForegroundColor Green + -+ # Deploy .gitattributes and .gitignore to project root if they don't exist -+ Write-Host "`n=== Deploying .gitattributes and .gitignore to project root ===" -ForegroundColor Magenta ++ # Deploy .gitattributes and .gitignore to project root if they don't exist ++ Write-Host "`n=== Deploying .gitattributes and .gitignore to project root ===" -ForegroundColor Magenta + $gitFiles = @( -+ @{ Name = '.gitattributes'; Source = Join-Path -Path $structDir -ChildPath 'components\gitattributes' }, -+ @{ Name = '.gitignore'; Source = Join-Path -Path $structDir -ChildPath 'components\gitignore' } ++ @{ Name = '.gitattributes'; Source = Join-Path -Path $structDir -ChildPath 'components\gitattributes' }, ++ @{ Name = '.gitignore'; Source = Join-Path -Path $structDir -ChildPath 'components\gitignore' } + ) + foreach ($gitFile in $gitFiles) { + $targetPath = Join-Path -Path $resolvedProject -ChildPath $gitFile.Name + try { + Copy-Item -Path $gitFile.Source -Destination $targetPath -Force -+ Write-Host "[OK] $targetPath" -ForegroundColor Green ++ Write-Host "[OK] $targetPath" -ForegroundColor Green + } + catch { -+ Write-Host "[FAIL] $targetPath" -ForegroundColor Red -+ Write-Host " $($_.Exception.Message)" -ForegroundColor DarkRed ++ Write-Host "[FAIL] $targetPath" -ForegroundColor Red ++ Write-Host " $($_.Exception.Message)" -ForegroundColor DarkRed + exit 1 + } + } @@ -446,7 +446,7 @@ Updating the script to set the hidden attribute on `.gitattributes` and `.gitign -
Tool use: **code_edit** • Edit file: ConfigLoader.ps1 +
Tool use: **search_replace** • Edit file: ConfigLoader.ps1 **Chunk 1** Lines added: 3, lines removed: undefined @@ -458,7 +458,7 @@ Lines added: 3, lines removed: undefined + # Set hidden attribute on the file + $file = Get-Item -LiteralPath $targetPath -Force + $file.Attributes = $file.Attributes -bor [System.IO.FileAttributes]::Hidden - Write-Host "[OK] $targetPath" -ForegroundColor Green + Write-Host "[OK] $targetPath" -ForegroundColor Green } ``` @@ -504,7 +504,7 @@ Updating the recursive update section to set the hidden attribute on `.gitattrib -
Tool use: **code_edit** • Edit file: ConfigLoader.ps1 +
Tool use: **search_replace** • Edit file: ConfigLoader.ps1 **Chunk 1** Lines added: 5, lines removed: undefined @@ -514,11 +514,11 @@ Lines added: 5, lines removed: undefined try { Copy-Item -Path $spec.Source -Destination $target.FullName -Force + # Set hidden attribute for .gitattributes and .gitignore files -+ if ($spec.Name -eq '.gitattributes' -or $spec.Name -eq '.gitignore') { ++ if ($spec.Name -eq '.gitattributes' -or $spec.Name -eq '.gitignore') { + $file = Get-Item -LiteralPath $target.FullName -Force + $file.Attributes = $file.Attributes -bor [System.IO.FileAttributes]::Hidden + } - Write-Host "[OK] $($target.FullName)" -ForegroundColor Green + Write-Host "[OK] $($target.FullName)" -ForegroundColor Green $updated++ ``` @@ -607,7 +607,7 @@ Updating the code to ignore `_CURRENT` folders: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -616,8 +616,8 @@ Lines added: 1, lines removed: 1 @@ -123,5 +123,5 @@ def is_archive_path(path: Path) -> bool: -- return any(part == "_archive" for part in path.parts) -+ return any(part in ("_archive", "_CURRENT") for part in path.parts) +- return any(part == "_archive" for part in path.parts) ++ return any(part in ("_archive", "_CURRENT") for part in path.parts) ``` @@ -630,7 +630,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -639,8 +639,8 @@ Lines added: 1, lines removed: 1 @@ -129,5 +129,5 @@ for dirpath, dirnames, filenames in os.walk(root): path = Path(dirpath) -- dirnames[:] = [d for d in dirnames if d != "_archive"] -+ dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] +- dirnames[:] = [d for d in dirnames if d != "_archive"] ++ dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] if is_archive_path(path): continue ``` @@ -653,7 +653,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -662,8 +662,8 @@ Lines added: 1, lines removed: 1 @@ -140,5 +140,5 @@ for dirpath, dirnames, filenames in os.walk(seq_dir): path = Path(dirpath) -- dirnames[:] = [d for d in dirnames if d != "_archive"] -+ dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] +- dirnames[:] = [d for d in dirnames if d != "_archive"] ++ dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] if is_archive_path(path): continue ``` @@ -853,57 +853,57 @@ Parameters: ```json { - "plan": "# Add RAM-Based Worker Limiting to zip_sequences.py\n\n## Overview\nModify `zip_sequences.py` to limit concurrent compression jobs based on available RAM. Use a hybrid approach: estimate RAM per job from folder size for 7z compression, with fallback to simple per-job estimates.\n\n## Implementation Details\n\n### 1. Add RAM detection function\n- Create `get_available_ram()` function in `zip_sequences.py`\n- Use `psutil` if available (cross-platform), fallback to platform-specific methods (Windows: `ctypes` with `GlobalMemoryStatusEx`)\n- Return available RAM in bytes\n- Reserve ~20% of total RAM for system (leaving 80% usable, but accounting for multiple 7z instances)\n\n### 2. Estimate RAM usage per job\n- Create `estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int` function\n- For 7z: estimate based on folder size (total bytes from `seq_state`)\n - Formula: `max(base_ram, folder_size_bytes * compression_factor)`\n - Base RAM: ~500MB per job\n - Compression factor: ~0.1-0.2x for 7z (7z uses ~80% of available, but we need to account for multiple jobs)\n - For very large folders (>10GB), cap estimate at reasonable max (e.g., 8GB per job)\n- For zip: use conservative estimate (~1GB per job, zipfile is more memory-efficient)\n- Fallback: if estimation fails, use simple per-job estimate (2GB for 7z, 1GB for zip)\n\n### 3. Modify `max_workers()` function\n- Rename to `calculate_max_workers()` or extend to accept RAM constraints\n- Add parameters: `requested: int | None`, `work_items: list | None = None`\n- Calculate CPU-based limit (existing logic)\n- If `work_items` provided and using 7z:\n - Get available RAM\n - Estimate RAM per job for each work item\n - Calculate max workers: `min(cpu_limit, floor(available_ram / max(estimated_ram_per_job)))`\n - Ensure minimum of 1 worker\n- If using zip: apply conservative RAM limiting (e.g., max 4-6 workers regardless of CPU)\n- Return the minimum of CPU limit and RAM limit\n\n### 4. Update `run_zip()` function\n- Calculate work items first (existing logic)\n- Pass work items to `calculate_max_workers()` to get RAM-aware worker count\n- Log RAM-based worker limit if verbose\n- Use calculated worker count for ThreadPoolExecutor\n\n### 5. Handle missing dependencies\n- Make `psutil` optional (try/except import)\n- If `psutil` unavailable, use platform-specific fallback\n- Log warning if RAM detection fails, fall back to CPU-based limits\n\n## Files to Modify\n- `zip_sequences.py`: Add RAM detection, estimation, and worker limiting logic\n\n## Configuration\n- No config changes needed initially\n- Could add `maxRamUsage` or `ramReservePercent` to config.json later if needed\n\n## Testing Considerations\n- Test on systems with different RAM amounts (32GB, 64GB, 128GB+)\n- Test with large folders (>10GB) and small folders (<1GB)\n- Verify fallback behavior when psutil unavailable\n- Ensure minimum 1 worker is always used", - "name": "Add RAM-based worker limiting to zip_sequences", - "todos": [ + "plan": "# Add RAM-Based Worker Limiting to zip_sequences.py\n\n## Overview\nModify `zip_sequences.py` to limit concurrent compression jobs based on available RAM. Use a hybrid approach: estimate RAM per job from folder size for 7z compression, with fallback to simple per-job estimates.\n\n## Implementation Details\n\n### 1. Add RAM detection function\n- Create `get_available_ram()` function in `zip_sequences.py`\n- Use `psutil` if available (cross-platform), fallback to platform-specific methods (Windows: `ctypes` with `GlobalMemoryStatusEx`)\n- Return available RAM in bytes\n- Reserve ~20% of total RAM for system (leaving 80% usable, but accounting for multiple 7z instances)\n\n### 2. Estimate RAM usage per job\n- Create `estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int` function\n- For 7z: estimate based on folder size (total bytes from `seq_state`)\n - Formula: `max(base_ram, folder_size_bytes * compression_factor)`\n - Base RAM: ~500MB per job\n - Compression factor: ~0.1-0.2x for 7z (7z uses ~80% of available, but we need to account for multiple jobs)\n - For very large folders (>10GB), cap estimate at reasonable max (e.g., 8GB per job)\n- For zip: use conservative estimate (~1GB per job, zipfile is more memory-efficient)\n- Fallback: if estimation fails, use simple per-job estimate (2GB for 7z, 1GB for zip)\n\n### 3. Modify `max_workers()` function\n- Rename to `calculate_max_workers()` or extend to accept RAM constraints\n- Add parameters: `requested: int | None`, `work_items: list | None = None`\n- Calculate CPU-based limit (existing logic)\n- If `work_items` provided and using 7z:\n - Get available RAM\n - Estimate RAM per job for each work item\n - Calculate max workers: `min(cpu_limit, floor(available_ram / max(estimated_ram_per_job)))`\n - Ensure minimum of 1 worker\n- If using zip: apply conservative RAM limiting (e.g., max 4-6 workers regardless of CPU)\n- Return the minimum of CPU limit and RAM limit\n\n### 4. Update `run_zip()` function\n- Calculate work items first (existing logic)\n- Pass work items to `calculate_max_workers()` to get RAM-aware worker count\n- Log RAM-based worker limit if verbose\n- Use calculated worker count for ThreadPoolExecutor\n\n### 5. Handle missing dependencies\n- Make `psutil` optional (try/except import)\n- If `psutil` unavailable, use platform-specific fallback\n- Log warning if RAM detection fails, fall back to CPU-based limits\n\n## Files to Modify\n- `zip_sequences.py`: Add RAM detection, estimation, and worker limiting logic\n\n## Configuration\n- No config changes needed initially\n- Could add `maxRamUsage` or `ramReservePercent` to config.json later if needed\n\n## Testing Considerations\n- Test on systems with different RAM amounts (32GB, 64GB, 128GB+)\n- Test with large folders (>10GB) and small folders (<1GB)\n- Verify fallback behavior when psutil unavailable\n- Ensure minimum 1 worker is always used", + "name": "Add RAM-based worker limiting to zip_sequences", + "todos": [ { - "content": "Add get_available_ram() function with psutil support and Windows fallback", - "status": "pending", - "id": "add-ram-detection" + "content": "Add get_available_ram() function with psutil support and Windows fallback", + "status": "pending", + "id": "add-ram-detection" }, { - "content": "Add estimate_ram_per_job() function that estimates RAM based on folder size for 7z, conservative estimate for zip", - "status": "pending", - "id": "add-ram-estimation" + "content": "Add estimate_ram_per_job() function that estimates RAM based on folder size for 7z, conservative estimate for zip", + "status": "pending", + "id": "add-ram-estimation" }, { - "content": "Modify max_workers() to accept work_items and calculate RAM-aware worker limits", - "status": "pending", - "id": "modify-max-workers", - "dependencies": [ - "add-ram-detection", - "add-ram-estimation" + "content": "Modify max_workers() to accept work_items and calculate RAM-aware worker limits", + "status": "pending", + "id": "modify-max-workers", + "dependencies": [ + "add-ram-detection", + "add-ram-estimation" ] }, { - "content": "Update run_zip() to calculate work items first, then pass to max_workers() for RAM-aware limit", - "status": "pending", - "id": "update-run-zip", - "dependencies": [ - "modify-max-workers" + "content": "Update run_zip() to calculate work items first, then pass to max_workers() for RAM-aware limit", + "status": "pending", + "id": "update-run-zip", + "dependencies": [ + "modify-max-workers" ] }, { - "content": "Add verbose logging for RAM detection, estimation, and worker limit calculations", - "status": "pending", - "id": "add-logging", - "dependencies": [ - "modify-max-workers" + "content": "Add verbose logging for RAM detection, estimation, and worker limit calculations", + "status": "pending", + "id": "add-logging", + "dependencies": [ + "modify-max-workers" ] } ], - "overview": "Implement hybrid RAM-based worker limiting that estimates RAM usage per job based on folder size (for 7z) and dynamically adjusts concurrent workers to prevent RAM exhaustion on systems with limited memory." + "overview": "Implement hybrid RAM-based worker limiting that estimates RAM usage per job based on folder size (for 7z) and dynamically adjusts concurrent workers to prevent RAM exhaustion on systems with limited memory." } ``` Additional data: ```json { - "status": "accepted", - "reviewData": { - "status": "Requested", - "selectedOption": "none", - "isShowingInput": false + "status": "accepted", + "reviewData": { + "status": "Requested", + "selectedOption": "none", + "isShowingInput": false } } ``` @@ -911,41 +911,41 @@ Result: ```json { - "accepted": { - "finalTodos": [ + "accepted": { + "finalTodos": [ { - "content": "Add get_available_ram() function with psutil support and Windows fallback", - "status": "pending", - "id": "add-ram-detection" + "content": "Add get_available_ram() function with psutil support and Windows fallback", + "status": "pending", + "id": "add-ram-detection" }, { - "content": "Add estimate_ram_per_job() function that estimates RAM based on folder size for 7z, conservative estimate for zip", - "status": "pending", - "id": "add-ram-estimation" + "content": "Add estimate_ram_per_job() function that estimates RAM based on folder size for 7z, conservative estimate for zip", + "status": "pending", + "id": "add-ram-estimation" }, { - "content": "Modify max_workers() to accept work_items and calculate RAM-aware worker limits", - "status": "pending", - "id": "modify-max-workers", - "dependencies": [ - "add-ram-detection", - "add-ram-estimation" + "content": "Modify max_workers() to accept work_items and calculate RAM-aware worker limits", + "status": "pending", + "id": "modify-max-workers", + "dependencies": [ + "add-ram-detection", + "add-ram-estimation" ] }, { - "content": "Update run_zip() to calculate work items first, then pass to max_workers() for RAM-aware limit", - "status": "pending", - "id": "update-run-zip", - "dependencies": [ - "modify-max-workers" + "content": "Update run_zip() to calculate work items first, then pass to max_workers() for RAM-aware limit", + "status": "pending", + "id": "update-run-zip", + "dependencies": [ + "modify-max-workers" ] }, { - "content": "Add verbose logging for RAM detection, estimation, and worker limit calculations", - "status": "pending", - "id": "add-logging", - "dependencies": [ - "modify-max-workers" + "content": "Add verbose logging for RAM detection, estimation, and worker limit calculations", + "status": "pending", + "id": "add-logging", + "dependencies": [ + "modify-max-workers" ] } ] @@ -1014,7 +1014,7 @@ Implementing step by step. Adding the RAM detection function first: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: undefined @@ -1042,7 +1042,7 @@ Lines added: 17, lines removed: undefined + except ImportError: + HAS_PSUTIL = False + # For Windows fallback -+ if platform.system() == "Windows": ++ if platform.system() == "Windows": + try: + import ctypes + from ctypes import wintypes @@ -1053,7 +1053,7 @@ Lines added: 17, lines removed: undefined + HAS_CTYPES = False + - RENDER_ROOT = Path("Renders") + RENDER_ROOT = Path("Renders") ```
@@ -1074,7 +1074,7 @@ Adding the `get_available_ram()` function: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 45, lines removed: undefined @@ -1085,11 +1085,11 @@ Lines added: 45, lines removed: undefined + + def get_available_ram() -> int | None: -+ """Get available RAM in bytes, reserving 20% for system. ++ """Get available RAM in bytes, reserving 20% for system. + + Returns: + Available RAM in bytes, or None if detection fails. -+ """ ++ """ + try: + if HAS_PSUTIL: + # Use psutil for cross-platform RAM detection @@ -1097,19 +1097,19 @@ Lines added: 45, lines removed: undefined + # Reserve 20% for system, use 80% for compression jobs + available = int(mem.total * 0.8) + return available -+ elif HAS_CTYPES and platform.system() == "Windows": ++ elif HAS_CTYPES and platform.system() == "Windows": + # Windows fallback using ctypes + class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ -+ ("dwLength", ctypes.c_ulong), -+ ("dwMemoryLoad", ctypes.c_ulong), -+ ("ullTotalPhys", ctypes.c_ulonglong), -+ ("ullAvailPhys", ctypes.c_ulonglong), -+ ("ullTotalPageFile", ctypes.c_ulonglong), -+ ("ullAvailPageFile", ctypes.c_ulonglong), -+ ("ullTotalVirtual", ctypes.c_ulonglong), -+ ("ullAvailVirtual", ctypes.c_ulonglong), -+ ("ullAvailExtendedVirtual", ctypes.c_ulonglong), ++ ("dwLength", ctypes.c_ulong), ++ ("dwMemoryLoad", ctypes.c_ulong), ++ ("ullTotalPhys", ctypes.c_ulonglong), ++ ("ullAvailPhys", ctypes.c_ulonglong), ++ ("ullTotalPageFile", ctypes.c_ulonglong), ++ ("ullAvailPageFile", ctypes.c_ulonglong), ++ ("ullTotalVirtual", ctypes.c_ulonglong), ++ ("ullAvailVirtual", ctypes.c_ulonglong), ++ ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + kernel32 = ctypes.windll.kernel32 @@ -1150,7 +1150,7 @@ Adding the `estimate_ram_per_job()` function: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 38, lines removed: undefined @@ -1161,7 +1161,7 @@ Lines added: 38, lines removed: undefined + + def estimate_ram_per_job(seq_dir: Path, seq_state: dict) -> int: -+ """Estimate RAM usage per compression job based on folder size. ++ """Estimate RAM usage per compression job based on folder size. + + Args: + seq_dir: Path to the sequence directory @@ -1169,16 +1169,16 @@ Lines added: 38, lines removed: undefined + + Returns: + Estimated RAM usage in bytes -+ """ ++ """ + # Calculate total folder size from seq_state -+ total_bytes = sum(entry.get("size", 0) for entry in seq_state.get("files", [])) ++ total_bytes = sum(entry.get("size", 0) for entry in seq_state.get("files", [])) + -+ if ZIPPER_TYPE == "7z": ++ if ZIPPER_TYPE == "7z": + # Base RAM: 500MB per job + base_ram = 500 * 1024 * 1024 # 500 MB + + # Compression factor: 7z can use significant RAM, especially for large files -+ # Use 0.15x factor (conservative estimate accounting for 7z's 80% usage) ++ # Use 0.15x factor (conservative estimate accounting for 7z's 80% usage) + compression_factor = 0.15 + + # For very large folders (>10GB), cap at 8GB per job @@ -1313,7 +1313,7 @@ Modifying `max_workers()` to accept work_items and calculate RAM-aware limits: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 73, lines removed: 4 @@ -1329,7 +1329,7 @@ Lines added: 73, lines removed: 4 + *, + verbose: bool = False + ) -> int: -+ """Calculate maximum worker count based on CPU and RAM constraints. ++ """Calculate maximum worker count based on CPU and RAM constraints. + + Args: + requested: User-requested worker count (from --jobs) @@ -1338,7 +1338,7 @@ Lines added: 73, lines removed: 4 + + Returns: + Maximum worker count (minimum of CPU and RAM limits) -+ """ ++ """ cpu = os.cpu_count() or 1 - limit = max(1, min(8, cpu)) + cpu_limit = max(1, min(8, cpu)) @@ -1357,7 +1357,7 @@ Lines added: 73, lines removed: 4 + if available_ram is None: + # RAM detection failed, fall back to CPU limit + if verbose: -+ log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) ++ log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) + return cpu_limit + + # Estimate RAM per job for each work item @@ -1368,7 +1368,7 @@ Lines added: 73, lines removed: 4 + ram_estimates.append(estimated_ram) + except Exception: + # If estimation fails, use fallback estimates -+ if ZIPPER_TYPE == "7z": ++ if ZIPPER_TYPE == "7z": + ram_estimates.append(2 * 1024 * 1024 * 1024) # 2GB fallback for 7z + else: + ram_estimates.append(1024 * 1024 * 1024) # 1GB fallback for zip @@ -1383,7 +1383,7 @@ Lines added: 73, lines removed: 4 + ram_limit = max(1, available_ram // max_ram_per_job) + + # For zip compression, apply conservative limit (max 6 workers) -+ if ZIPPER_TYPE == "zip": ++ if ZIPPER_TYPE == "zip": + ram_limit = min(ram_limit, 6) + + # Return minimum of CPU and RAM limits @@ -1393,9 +1393,9 @@ Lines added: 73, lines removed: 4 + ram_gb = available_ram / (1024 ** 3) + max_ram_gb = max_ram_per_job / (1024 ** 3) + log( -+ "zip", -+ f"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → " -+ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", ++ "zip", ++ f"RAM: {ram_gb:.1f}GB available, ~{max_ram_gb:.1f}GB per job → " ++ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + verbose_only=True, + verbose=verbose + ) @@ -1437,7 +1437,7 @@ Updating `run_zip()` to calculate work items first, then pass them to `max_worke -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1498,7 +1498,7 @@ Updating `main()` to pass the requested workers count correctly: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 4, lines removed: 2 @@ -1509,7 +1509,7 @@ Lines added: 4, lines removed: 2 args = parse_args() - workers = max_workers(args.jobs) - if args.mode == "expand": + if args.mode == "expand": + # For expand mode, use simple CPU-based worker calculation + workers = max_workers(args.jobs, work_items=None, verbose=args.verbose) run_expand(workers, verbose=args.verbose) @@ -1617,7 +1617,7 @@ Removing the unused `wintypes` import: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: undefined, lines removed: 1 @@ -1796,7 +1796,7 @@ Adding per-instance memory limits to 7z. Updating the code to calculate and appl -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1807,7 +1807,7 @@ Lines added: 1, lines removed: 1 verbose: bool = False - ) -> int: + ) -> tuple[int, int | None]: - """Calculate maximum worker count based on CPU and RAM constraints. + """Calculate maximum worker count based on CPU and RAM constraints. ``` @@ -1819,7 +1819,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 2, lines removed: 1 @@ -1831,7 +1831,7 @@ Lines added: 2, lines removed: 1 - Maximum worker count (minimum of CPU and RAM limits) + Tuple of (worker_count, per_job_memory_limit_bytes) + per_job_memory_limit_bytes is None if not using 7z or RAM detection failed - """ + """ cpu = os.cpu_count() or 1 ``` @@ -1843,7 +1843,7 @@ Lines added: 2, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1866,7 +1866,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1874,7 +1874,7 @@ Lines added: 1, lines removed: 1 ```diff @@ -241,5 +241,5 @@ if verbose: - log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) + log("zip", "RAM detection failed, using CPU-based worker limit", verbose_only=True, verbose=verbose) - return cpu_limit + return (cpu_limit, None) @@ -1889,7 +1889,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1913,7 +1913,7 @@ Lines added: 9, lines removed: undefined + # Calculate per-job memory limit for 7z (divide available RAM by worker count with safety margin) + per_job_memory_limit = None -+ if ZIPPER_TYPE == "7z" and available_ram is not None: ++ if ZIPPER_TYPE == "7z" and available_ram is not None: + # Allocate available RAM across workers, leaving 10% safety margin + per_job_memory_limit = int((available_ram * 0.9) // final_limit) + # Ensure minimum of 1GB and maximum reasonable limit @@ -1924,7 +1924,7 @@ Lines added: 9, lines removed: undefined max_ram_gb = max_ram_per_job / (1024 ** 3) + mem_limit_gb = (per_job_memory_limit / (1024 ** 3)) if per_job_memory_limit else 0 log( - "zip", + "zip", ``` **Chunk 3** @@ -1938,8 +1938,8 @@ Lines added: 9, lines removed: 2 - return final_limit + if per_job_memory_limit: + log( -+ "zip", -+ f"7z memory limit per job: {mem_limit_gb:.1f}GB", ++ "zip", ++ f"7z memory limit per job: {mem_limit_gb:.1f}GB", + verbose_only=True, + verbose=verbose + ) @@ -1967,7 +1967,7 @@ Updating `run_zip()` to handle the tuple return and pass the memory limit: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -1990,7 +1990,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -2013,7 +2013,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 2, lines removed: 2 @@ -2024,7 +2024,7 @@ Lines added: 2, lines removed: 2 - def process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, *, verbose: bool) -> Sequence[Path]: + def process_zip(seq_dir: Path, zip_path: Path, state_path: Path, seq_state: dict, per_job_memory_limit: int | None, *, verbose: bool) -> Sequence[Path]: - log("zip", f"{seq_dir} -> {zip_path}", verbose_only=True, verbose=verbose) + log("zip", f"{seq_dir} -> {zip_path}", verbose_only=True, verbose=verbose) - zip_sequence(seq_dir, zip_path) + zip_sequence(seq_dir, zip_path, per_job_memory_limit) state_path.write_text(json.dumps(seq_state, indent=2)) @@ -2039,7 +2039,7 @@ Lines added: 2, lines removed: 2 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -2050,7 +2050,7 @@ Lines added: 1, lines removed: 1 - def zip_sequence(seq_dir: Path, zip_path: Path) -> None: + def zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None = None) -> None: - if ZIPPER_TYPE == "7z": + if ZIPPER_TYPE == "7z": if SEVEN_Z_EXE is None: ``` @@ -2062,15 +2062,15 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 11, lines removed: 1 ```diff @@ -445,7 +445,17 @@ - f"-mx={COMPRESSION_LEVEL}", - "-t7z", # Use 7z format, not zip + f"-mx={COMPRESSION_LEVEL}", + "-t7z", # Use 7z format, not zip + ] + + # Add memory limit if specified (7z -mmem flag limits memory usage) @@ -2078,11 +2078,11 @@ Lines added: 11, lines removed: 1 + # -mmem format: -mmem=Size[b|k|m|g] or -mmem=Size% (percentage) + # Convert bytes to MB for 7z + mem_mb = per_job_memory_limit // (1024 * 1024) -+ cmd.append(f"-mmem={mem_mb}m") ++ cmd.append(f"-mmem={mem_mb}m") + + cmd.extend([ str(temp_zip_abs), - f"@{list_file_abs}", + f"@{list_file_abs}", - ] + ]) result = subprocess.run( @@ -2107,14 +2107,14 @@ Updating `main()` to handle the tuple return: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 ```diff @@ -759,5 +759,5 @@ - if args.mode == "expand": + if args.mode == "expand": # For expand mode, use simple CPU-based worker calculation - workers = max_workers(args.jobs, work_items=None, verbose=args.verbose) + workers, _ = max_workers(args.jobs, work_items=None, verbose=args.verbose) @@ -2341,7 +2341,7 @@ The `-mmem` flag may be invalid or causing 7z to fail silently. Checking 7z comm -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 6, lines removed: 4 @@ -2358,10 +2358,10 @@ Lines added: 6, lines removed: 4 - # Convert bytes to MB for 7z + # Convert bytes to MB, then use ~1/3 for dictionary (leaving room for other operations) mem_mb = per_job_memory_limit // (1024 * 1024) -- cmd.append(f"-mmem={mem_mb}m") +- cmd.append(f"-mmem={mem_mb}m") + # Use dictionary size of ~1/3 of available memory, capped at reasonable limits + dict_size_mb = max(32, min(mem_mb // 3, 2048)) # Min 32MB, max 2GB -+ cmd.append(f"-md={dict_size_mb}m") ++ cmd.append(f"-md={dict_size_mb}m") cmd.extend([ ``` @@ -2418,7 +2418,7 @@ Updating verbose logging to show the dictionary size being used: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 4, lines removed: 1 @@ -2431,9 +2431,9 @@ Lines added: 4, lines removed: 1 + mem_mb = per_job_memory_limit // (1024 * 1024) + dict_size_mb = max(32, min(mem_mb // 3, 2048)) log( - "zip", -- f"7z memory limit per job: {mem_limit_gb:.1f}GB", -+ f"7z memory limit per job: {mem_limit_gb:.1f}GB (dictionary size: {dict_size_mb}MB)", + "zip", +- f"7z memory limit per job: {mem_limit_gb:.1f}GB", ++ f"7z memory limit per job: {mem_limit_gb:.1f}GB (dictionary size: {dict_size_mb}MB)", verbose_only=True, verbose=verbose ``` @@ -2535,7 +2535,7 @@ Reducing the dictionary size cap and increasing the safety margin: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 3, lines removed: 3 @@ -2543,7 +2543,7 @@ Lines added: 3, lines removed: 3 ```diff @@ -275,8 +275,8 @@ per_job_memory_limit = None - if ZIPPER_TYPE == "7z" and available_ram is not None: + if ZIPPER_TYPE == "7z" and available_ram is not None: - # Allocate available RAM across workers, leaving 10% safety margin - per_job_memory_limit = int((available_ram * 0.9) // final_limit) + # Allocate available RAM across workers, leaving 25% safety margin for system and overhead @@ -2563,7 +2563,7 @@ Lines added: 3, lines removed: 3 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 3, lines removed: 3 @@ -2579,7 +2579,7 @@ Lines added: 3, lines removed: 3 - dict_size_mb = max(32, min(mem_mb // 3, 2048)) # Min 32MB, max 2GB + # Use dictionary size of ~1/4 of available memory, capped at 1GB for safety + dict_size_mb = max(32, min(mem_mb // 4, 1024)) # Min 32MB, max 1GB - cmd.append(f"-md={dict_size_mb}m") + cmd.append(f"-md={dict_size_mb}m") ``` @@ -2591,7 +2591,7 @@ Lines added: 3, lines removed: 3 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -2603,7 +2603,7 @@ Lines added: 1, lines removed: 1 - dict_size_mb = max(32, min(mem_mb // 3, 2048)) + dict_size_mb = max(32, min(mem_mb // 4, 1024)) log( - "zip", + "zip", ```
@@ -2769,7 +2769,7 @@ Updating the code to use a fixed dictionary size and calculate workers from 80% -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 36, lines removed: 18 @@ -2780,7 +2780,7 @@ Lines added: 36, lines removed: 18 - # Use the maximum estimated RAM per job (worst case) + # For 7z: use fixed dictionary size and calculate workers based on 80% RAM allocation -+ if ZIPPER_TYPE == "7z" and available_ram is not None: ++ if ZIPPER_TYPE == "7z" and available_ram is not None: + # Fixed dictionary size: 1GB (1024MB) + FIXED_DICT_SIZE_MB = 1024 + # 7z uses ~2-3x dictionary size in RAM, use 3x for safety @@ -2803,10 +2803,10 @@ Lines added: 36, lines removed: 18 + compression_ram_gb = compression_ram / (1024 ** 3) + ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) + log( -+ "zip", -+ f"RAM: {ram_gb:.1f}GB total, {compression_ram_gb:.1f}GB for compression (80%) → " -+ f"{ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", ++ "zip", ++ f"RAM: {ram_gb:.1f}GB total, {compression_ram_gb:.1f}GB for compression (80%) → " ++ f"{ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + verbose_only=True, + verbose=verbose + ) @@ -2820,7 +2820,7 @@ Lines added: 36, lines removed: 18 ram_limit = max(1, available_ram // max_ram_per_job) - - # For zip compression, apply conservative limit (max 6 workers) -- if ZIPPER_TYPE == "zip": +- if ZIPPER_TYPE == "zip": - ram_limit = min(ram_limit, 6) - - # Return minimum of CPU and RAM limits @@ -2829,7 +2829,7 @@ Lines added: 36, lines removed: 18 - # Calculate per-job memory limit for 7z (divide available RAM by worker count with safety margin) - per_job_memory_limit = None -- if ZIPPER_TYPE == "7z" and available_ram is not None: +- if ZIPPER_TYPE == "7z" and available_ram is not None: - # Allocate available RAM across workers, leaving 25% safety margin for system and overhead - per_job_memory_limit = int((available_ram * 0.75) // final_limit) - # Ensure minimum of 1GB and maximum reasonable limit @@ -2840,7 +2840,7 @@ Lines added: 36, lines removed: 18 max_ram_gb = max_ram_per_job / (1024 ** 3) - mem_limit_gb = (per_job_memory_limit / (1024 ** 3)) if per_job_memory_limit else 0 log( - "zip", + "zip", ``` **Chunk 2** @@ -2855,8 +2855,8 @@ Lines added: 2, lines removed: 12 - mem_mb = per_job_memory_limit // (1024 * 1024) - dict_size_mb = max(32, min(mem_mb // 4, 1024)) - log( -- "zip", -- f"7z memory limit per job: {mem_limit_gb:.1f}GB (dictionary size: {dict_size_mb}MB)", +- "zip", +- f"7z memory limit per job: {mem_limit_gb:.1f}GB (dictionary size: {dict_size_mb}MB)", - verbose_only=True, - verbose=verbose - ) @@ -2876,7 +2876,7 @@ Lines added: 2, lines removed: 12 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 3, lines removed: 6 @@ -2895,7 +2895,7 @@ Lines added: 3, lines removed: 6 - dict_size_mb = max(32, min(mem_mb // 4, 1024)) # Min 32MB, max 1GB + # per_job_memory_limit is actually the fixed dictionary size in bytes + dict_size_mb = per_job_memory_limit // (1024 * 1024) - cmd.append(f"-md={dict_size_mb}m") + cmd.append(f"-md={dict_size_mb}m") ``` @@ -2951,7 +2951,7 @@ Fixing the calculation: `get_available_ram()` already returns 80% of total RAM ( -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -2962,7 +2962,7 @@ Lines added: 1, lines removed: 1 - # For 7z: use fixed dictionary size and calculate workers based on 80% RAM allocation + # For 7z: use fixed dictionary size and calculate workers based on available RAM - if ZIPPER_TYPE == "7z" and available_ram is not None: + if ZIPPER_TYPE == "7z" and available_ram is not None: # Fixed dictionary size: 1GB (1024MB) ``` @@ -2990,7 +2990,7 @@ Lines added: 3, lines removed: 2 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 2 @@ -2998,11 +2998,11 @@ Lines added: 1, lines removed: 2 ```diff @@ -285,6 +285,5 @@ log( - "zip", -- f"RAM: {ram_gb:.1f}GB total, {compression_ram_gb:.1f}GB for compression (80%) → " -- f"{ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " - f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + "zip", +- f"RAM: {ram_gb:.1f}GB total, {compression_ram_gb:.1f}GB for compression (80%) → " +- f"{ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", verbose_only=True, ``` @@ -3051,7 +3051,7 @@ Removing the redundant variable from the log message: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: undefined, lines removed: 1 @@ -3201,7 +3201,7 @@ Optimizing initialization: for 7z, skip RAM estimation (we use fixed values), an -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 16 @@ -3218,7 +3218,7 @@ Lines added: 1, lines removed: 16 - ram_estimates.append(estimated_ram) - except Exception: - # If estimation fails, use fallback estimates -- if ZIPPER_TYPE == "7z": +- if ZIPPER_TYPE == "7z": - ram_estimates.append(2 * 1024 * 1024 * 1024) # 2GB fallback for 7z - else: - ram_estimates.append(1024 * 1024 * 1024) # 1GB fallback for zip @@ -3228,7 +3228,7 @@ Lines added: 1, lines removed: 16 - # For 7z: use fixed dictionary size and calculate workers based on available RAM + # Skip RAM estimation loop for 7z since we use fixed values - if ZIPPER_TYPE == "7z" and available_ram is not None: + if ZIPPER_TYPE == "7z" and available_ram is not None: # Fixed dictionary size: 1GB (1024MB) ``` @@ -3240,7 +3240,7 @@ Lines added: 1, lines removed: 16 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 13, lines removed: undefined @@ -3284,7 +3284,7 @@ Optimizing `run_zip()` initialization: check existing archives first, then compu -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 31, lines removed: 13 @@ -3294,10 +3294,10 @@ Lines added: 31, lines removed: 13 if RENDER_ROOT.exists(): for seq_dir in find_sequence_dirs(RENDER_ROOT): - seq_state = compute_state(seq_dir) -- if not seq_state["files"]: +- if not seq_state["files"]: - continue - - # Get the target archive path (will be .7z if ZIPPER_TYPE is "7z") + # Get the target archive path (will be .7z if ZIPPER_TYPE is "7z") zip_path = archive_path_for(seq_dir) state_path = state_path_for(zip_path) + @@ -3306,16 +3306,16 @@ Lines added: 31, lines removed: 13 # Check if we need to upgrade from .zip to .7z old_zip_path = None - if ZIPPER_TYPE == "7z": + if ZIPPER_TYPE == "7z": - # Check if an old .zip file exists - old_zip_path = zip_path.with_suffix(".zip") + old_zip_path = zip_path.with_suffix(".zip") if old_zip_path.exists(): -- # Check if the old .zip's metadata matches current state +- # Check if the old .zip's metadata matches current state old_state_path = state_path_for(old_zip_path) old_stored_state = load_state(old_state_path) - if not state_changed(seq_state, old_stored_state): - # Old .zip is up to date, skip conversion -+ # If old .zip exists and archive doesn't, we'll check state later ++ # If old .zip exists and archive doesn't, we'll check state later + if not zip_path.exists() and old_stored_state is not None: + stored_state = old_stored_state + @@ -3325,7 +3325,7 @@ Lines added: 31, lines removed: 13 + try: + dir_mtime = seq_dir.stat().st_mtime_ns + archive_mtime = zip_path.stat().st_mtime_ns -+ # If directory wasn't modified since archive was created, skip state computation ++ # If directory wasn't modified since archive was created, skip state computation + if dir_mtime <= archive_mtime: + # Still need to check for old .zip cleanup + if old_zip_path and old_zip_path.exists(): @@ -3344,7 +3344,7 @@ Lines added: 31, lines removed: 13 - if not state_changed(seq_state, stored_state): + # Compute current state only if we need to + seq_state = compute_state(seq_dir) -+ if not seq_state["files"]: ++ if not seq_state["files"]: + continue + + # Check if state changed @@ -3454,7 +3454,7 @@ Making the limits stricter: increasing the RAM multiplier and reducing the RAM a -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 4, lines removed: 4 @@ -3485,7 +3485,7 @@ Lines added: 4, lines removed: 4 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 2, lines removed: 1 @@ -3497,10 +3497,10 @@ Lines added: 2, lines removed: 1 + compression_ram_gb = compression_ram / (1024 ** 3) ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) log( - "zip", -- f"RAM: {ram_gb:.1f}GB available (80% of total), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (70%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " - f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (70%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", verbose_only=True, ``` @@ -3603,7 +3603,7 @@ Reducing RAM allocation more aggressively: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 4, lines removed: 4 @@ -3634,7 +3634,7 @@ Lines added: 4, lines removed: 4 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -3642,10 +3642,10 @@ Lines added: 1, lines removed: 1 ```diff @@ -270,5 +270,5 @@ log( - "zip", -- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (70%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (50%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " - f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (70%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (50%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", verbose_only=True, ``` @@ -3729,7 +3729,7 @@ Making the limits more aggressive. Checking the current calculation and tighteni -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 5, lines removed: 4 @@ -3741,7 +3741,7 @@ Lines added: 5, lines removed: 4 - # 7z uses ~2-3x dictionary size in RAM, but with overhead use 6x for safety - FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 6 * 1024 * 1024 # 6GB per job + # 7z uses ~2-3x dictionary size in RAM, but with overhead use 8x for safety -+ # This accounts for 7z's internal buffers, OS overhead, and other processes ++ # This accounts for 7z's internal buffers, OS overhead, and other processes + FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 8 * 1024 * 1024 # 8GB per job # available_ram is already 80% of total (20% reserved for system) @@ -3761,7 +3761,7 @@ Lines added: 5, lines removed: 4 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 @@ -3769,10 +3769,10 @@ Lines added: 1, lines removed: 1 ```diff @@ -271,5 +271,5 @@ log( - "zip", -- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (50%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " - f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (50%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", verbose_only=True, ``` @@ -3874,16 +3874,16 @@ Adding the config option and using it to limit 7z instances: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: undefined ```diff @@ -55,4 +55,5 @@ - "compression": 9, - "dailyFormat": "daily_YYMMDD", -+ "Max7zInst": None, # Maximum concurrent 7z instances (None = auto-calculate) + "compression": 9, + "dailyFormat": "daily_YYMMDD", ++ "Max7zInst": None, # Maximum concurrent 7z instances (None = auto-calculate) } ``` @@ -3904,7 +3904,7 @@ Lines added: 1, lines removed: undefined -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 10, lines removed: undefined @@ -3913,7 +3913,7 @@ Lines added: 10, lines removed: undefined @@ -108,4 +108,14 @@ COMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL)) -+ MAX_7Z_INSTANCES = CONFIG.get("Max7zInst") ++ MAX_7Z_INSTANCES = CONFIG.get("Max7zInst") + if MAX_7Z_INSTANCES is not None: + if isinstance(MAX_7Z_INSTANCES, str): + try: @@ -3924,7 +3924,7 @@ Lines added: 10, lines removed: undefined + MAX_7Z_INSTANCES = None + SEVEN_Z_EXE: str | None = None - if ZIPPER_TYPE == "7z": + if ZIPPER_TYPE == "7z": ```
@@ -3935,7 +3935,7 @@ Lines added: 10, lines removed: undefined -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 56, lines removed: 33 @@ -3946,13 +3946,13 @@ Lines added: 56, lines removed: 33 - # For 7z: use fixed dictionary size and calculate workers based on available RAM - # Skip RAM estimation loop for 7z since we use fixed values -- if ZIPPER_TYPE == "7z" and available_ram is not None: +- if ZIPPER_TYPE == "7z" and available_ram is not None: + # For 7z: use fixed dictionary size and calculate workers -+ if ZIPPER_TYPE == "7z": ++ if ZIPPER_TYPE == "7z": # Fixed dictionary size: 1GB (1024MB) FIXED_DICT_SIZE_MB = 1024 - # 7z uses ~2-3x dictionary size in RAM, but with overhead use 8x for safety -- # This accounts for 7z's internal buffers, OS overhead, and other processes +- # This accounts for 7z's internal buffers, OS overhead, and other processes - FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 8 * 1024 * 1024 # 8GB per job - - # available_ram is already 80% of total (20% reserved for system) @@ -3973,9 +3973,9 @@ Lines added: 56, lines removed: 33 - compression_ram_gb = compression_ram / (1024 ** 3) - ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) - log( -- "zip", -- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -- f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", +- "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " +- f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", - verbose_only=True, - verbose=verbose - ) @@ -3985,16 +3985,16 @@ Lines added: 56, lines removed: 33 + + # Check if Max7zInst is configured + if MAX_7Z_INSTANCES is not None: -+ # Use configured maximum instances, but still respect user's --jobs and CPU limit ++ # Use configured maximum instances, but still respect user's --jobs and CPU limit + final_limit = min(cpu_limit, MAX_7Z_INSTANCES) + if requested and requested > 0: + final_limit = min(final_limit, requested) + + if verbose: + log( -+ "zip", -+ f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " -+ f"CPU limit: {cpu_limit}, requested: {requested}, final: {final_limit}", ++ "zip", ++ f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " ++ f"CPU limit: {cpu_limit}, requested: {requested}, final: {final_limit}", + verbose_only=True, + verbose=verbose + ) @@ -4004,7 +4004,7 @@ Lines added: 56, lines removed: 33 + # Auto-calculate based on RAM if Max7zInst not configured + if available_ram is not None: + # 7z uses ~2-3x dictionary size in RAM, but with overhead use 8x for safety -+ # This accounts for 7z's internal buffers, OS overhead, and other processes ++ # This accounts for 7z's internal buffers, OS overhead, and other processes + FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 8 * 1024 * 1024 # 8GB per job + + # available_ram is already 80% of total (20% reserved for system) @@ -4022,9 +4022,9 @@ Lines added: 56, lines removed: 33 + compression_ram_gb = compression_ram / (1024 ** 3) + ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) + log( -+ "zip", -+ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -+ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", ++ "zip", ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", + verbose_only=True, + verbose=verbose + ) @@ -4033,7 +4033,7 @@ Lines added: 56, lines removed: 33 + + # RAM detection failed, fall back to CPU limit + if verbose: -+ log("zip", "RAM detection failed and Max7zInst not set, using CPU-based worker limit", verbose_only=True, verbose=verbose) ++ log("zip", "RAM detection failed and Max7zInst not set, using CPU-based worker limit", verbose_only=True, verbose=verbose) + return (cpu_limit, fixed_dict_size_bytes) # For zip compression, use existing estimation-based approach @@ -4107,18 +4107,18 @@ The string to replace was not found in the file. -
Tool use: **code_edit** • Edit file: config.json +
Tool use: **search_replace** • Edit file: config.json **Chunk 1** Lines added: 2, lines removed: 1 ```diff @@ -3,5 +3,6 @@ - "structDir": "D:\\0 ProjectStructure", - "zipper": "7z", -- "compression": 9 -+ "compression": 9, -+ "Max7zInst": null + "structDir": "D:\\0 ProjectStructure", + "zipper": "7z", +- "compression": 9 ++ "compression": 9, ++ "Max7zInst": null } ``` @@ -4188,7 +4188,7 @@ Removing CPU limits from the 7z worker calculation: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 2, lines removed: 2 @@ -4197,9 +4197,9 @@ Lines added: 2, lines removed: 2 @@ -262,6 +262,6 @@ # Check if Max7zInst is configured if MAX_7Z_INSTANCES is not None: -- # Use configured maximum instances, but still respect user's --jobs and CPU limit +- # Use configured maximum instances, but still respect user's --jobs and CPU limit - final_limit = min(cpu_limit, MAX_7Z_INSTANCES) -+ # Use configured maximum instances, but still respect user's --jobs ++ # Use configured maximum instances, but still respect user's --jobs + final_limit = MAX_7Z_INSTANCES if requested and requested > 0: final_limit = min(final_limit, requested) @@ -4210,10 +4210,10 @@ Lines added: 1, lines removed: 1 ```diff @@ -271,5 +271,5 @@ - "zip", - f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " -- f"CPU limit: {cpu_limit}, requested: {requested}, final: {final_limit}", -+ f"requested: {requested}, final: {final_limit}", + "zip", + f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " +- f"CPU limit: {cpu_limit}, requested: {requested}, final: {final_limit}", ++ f"requested: {requested}, final: {final_limit}", verbose_only=True, verbose=verbose ``` @@ -4226,7 +4226,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 4, lines removed: 2 @@ -4250,10 +4250,10 @@ Lines added: 1, lines removed: 1 ```diff @@ -301,5 +303,5 @@ - "zip", - f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " -- f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", -+ f"RAM limit: {ram_limit}, requested: {requested}, final: {final_limit}", + "zip", + f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " +- f"RAM limit: {ram_limit}, CPU limit: {cpu_limit}, final: {final_limit}", ++ f"RAM limit: {ram_limit}, requested: {requested}, final: {final_limit}", verbose_only=True, verbose=verbose ``` @@ -4268,9 +4268,9 @@ Lines added: 6, lines removed: 3 - # RAM detection failed, fall back to CPU limit + # RAM detection failed, use a safe default (no CPU limit) if verbose: -- log("zip", "RAM detection failed and Max7zInst not set, using CPU-based worker limit", verbose_only=True, verbose=verbose) +- log("zip", "RAM detection failed and Max7zInst not set, using CPU-based worker limit", verbose_only=True, verbose=verbose) - return (cpu_limit, fixed_dict_size_bytes) -+ log("zip", "RAM detection failed and Max7zInst not set, using default worker limit of 4", verbose_only=True, verbose=verbose) ++ log("zip", "RAM detection failed and Max7zInst not set, using default worker limit of 4", verbose_only=True, verbose=verbose) + default_limit = 4 + if requested and requested > 0: + default_limit = requested @@ -4377,14 +4377,14 @@ The issue is that `Get-ChildItem` doesn't find hidden files by default. Since `. -
Tool use: **code_edit** • Edit file: ConfigLoader.ps1 +
Tool use: **search_replace** • Edit file: ConfigLoader.ps1 **Chunk 1** Lines added: 1, lines removed: 1 ```diff @@ -256,5 +256,5 @@ - Write-Host "`n=== Updating $($spec.Name) ===" -ForegroundColor Magenta + Write-Host "`n=== Updating $($spec.Name) ===" -ForegroundColor Magenta - $targets = Get-ChildItem -LiteralPath $resolvedProject -Recurse -Filter $spec.Name -File -ErrorAction SilentlyContinue + $targets = Get-ChildItem -LiteralPath $resolvedProject -Recurse -Filter $spec.Name -File -Force -ErrorAction SilentlyContinue @@ -4480,17 +4480,17 @@ Updating the code to use `0` for auto-calculate and updating the config: -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 1, lines removed: 1 ```diff @@ -55,5 +55,5 @@ - "compression": 9, - "dailyFormat": "daily_YYMMDD", -- "Max7zInst": None, # Maximum concurrent 7z instances (None = auto-calculate) -+ "Max7zInst": 0, # Maximum concurrent 7z instances (0 = auto-calculate) + "compression": 9, + "dailyFormat": "daily_YYMMDD", +- "Max7zInst": None, # Maximum concurrent 7z instances (None = auto-calculate) ++ "Max7zInst": 0, # Maximum concurrent 7z instances (0 = auto-calculate) } ``` @@ -4503,7 +4503,7 @@ Lines added: 1, lines removed: 1 -
Tool use: **code_edit** • Edit file: zip_sequences.py +
Tool use: **search_replace** • Edit file: zip_sequences.py **Chunk 1** Lines added: 6, lines removed: 3 @@ -4512,8 +4512,8 @@ Lines added: 6, lines removed: 3 @@ -108,13 +108,16 @@ COMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL)) -- MAX_7Z_INSTANCES = CONFIG.get("Max7zInst") -+ MAX_7Z_INSTANCES = CONFIG.get("Max7zInst", 0) +- MAX_7Z_INSTANCES = CONFIG.get("Max7zInst") ++ MAX_7Z_INSTANCES = CONFIG.get("Max7zInst", 0) if MAX_7Z_INSTANCES is not None: if isinstance(MAX_7Z_INSTANCES, str): try: @@ -4539,17 +4539,17 @@ Lines added: 6, lines removed: 3 -
Tool use: **code_edit** • Edit file: config.json +
Tool use: **search_replace** • Edit file: config.json **Chunk 1** Lines added: 1, lines removed: 1 ```diff @@ -4,5 +4,5 @@ - "zipper": "7z", - "compression": 9, -- "Max7zInst": 1 -+ "Max7zInst": 0 + "zipper": "7z", + "compression": 9, +- "Max7zInst": 1 ++ "Max7zInst": 0 } ``` @@ -4889,7 +4889,6 @@ _**Agent (model default, mode Agent)**_ --- - Threadripper system with ~96GB RAM. The script is using it well: - **76.7GB available** (80% of total) diff --git a/components/gitignore b/components/gitignore index 242ab6a..f1ed702 100644 --- a/components/gitignore +++ b/components/gitignore @@ -123,4 +123,4 @@ Adobe Premiere Pro (Beta) Audio Previews/ footage/ plates/ -Pr/RnR/ \ No newline at end of file +RnR/ \ No newline at end of file diff --git a/config.json b/config.json index d6572d9..ff6cb79 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,6 @@ { "dailyFormat": "YYYY-MM-DD", - "structDir": "A:\\1 Amazon_Active_Projects\\3 ProjectStructure", + "structDir": "D:\\0 ProjectStructure", "zipper": "7z", "compression": 0, "Max7zInst": 0