From c101fbb3814dc7106867f6768eb7e910d9e4f531 Mon Sep 17 00:00:00 2001 From: Nathan Date: Tue, 25 Nov 2025 14:22:44 -0700 Subject: [PATCH] fix zipseq store not storing --- ...ix-archive-removal-issue-in-zipseqarchv.md | 1011 ++++++++++++++++- config.json | 2 +- zip_sequences.py | 118 +- 3 files changed, 1091 insertions(+), 40 deletions(-) diff --git a/.specstory/history/2025-11-25_20-03Z-fix-archive-removal-issue-in-zipseqarchv.md b/.specstory/history/2025-11-25_20-03Z-fix-archive-removal-issue-in-zipseqarchv.md index 0fa1de4..aefff28 100644 --- a/.specstory/history/2025-11-25_20-03Z-fix-archive-removal-issue-in-zipseqarchv.md +++ b/.specstory/history/2025-11-25_20-03Z-fix-archive-removal-issue-in-zipseqarchv.md @@ -1804,7 +1804,6 @@ _**Agent (model default, mode Agent)**_ --- - The fix is working as expected. The script now: 1. **Always runs cleanup** — even when no sequences need zipping @@ -1816,3 +1815,1013 @@ The cleanup function now runs in all cases, not just when sequences are queued f --- +_**User (2025-11-25 21:04Z)**_ + +another issue: it's ignoring the sequence within this folder: + +P:\251119_Tatt2Away_How-it-Works\Renders\scab_v2.1\ + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +Investigating why that sequence is being ignored. Checking how sequences are detected: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + +
+ Tool use: **codebase_search** • Searched codebase "How does find_sequence_dirs detect sequence directories and what conditions might cause a directory to be skipped?" • **14** results + +| File | Lines | +|------|-------| +| `zip_sequences.py` | L471-496 | +| `zip_sequences.py` | L783-940 | +| `zip_sequences.py` | L478-514 | +| `UpdateSequences.ps1` | L194-235 | +| `zip_sequences.py` | L785-940 | +| `UpdateSequences.ps1` | L232-276 | +| `zip_sequences.py` | L943-1070 | +| `UpdateSequences.ps1` | L60 | +| `zip_sequences.py` | L553-780 | +| `UpdateSequences.ps1` | L143-200 | +| `zip_sequences.py` | L102 | +| `zip_sequences.py` | L546-724 | +| `zip_sequences.py` | L987-1091 | +| `zip_sequences.py` | L788-1070 | + +
+ + +--- + + + +--- + + +Checking the sequence detection logic. `find_sequence_dirs` only detects directories that contain image files directly. If `scab_v2.1` has images only in subdirectories, it may be missed. Verifying and improving the detection: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +Adding debug logging to see why `scab_v2.1` is being skipped. Improving the detection logic to handle edge cases: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 17, lines removed: undefined + +```diff +@@ -486,4 +486,21 @@ + if has_frames: + yield path ++ # Also check subdirectories for sequence files (in case the directory itself has no files) ++ # but only if the directory is not too deep (to avoid scanning archive directories) ++ elif not any(part in ("_archive", "_CURRENT", "_zipped") for part in path.parts): ++ # Check if any immediate subdirectory has sequence files ++ for subdir in dirnames: ++ subdir_path = path / subdir ++ if subdir_path.is_dir(): ++ try: ++ subdir_files = list(subdir_path.iterdir()) ++ if any(f.is_file() and f.suffix.lower() in SEQUENCE_EXTENSIONS for f in subdir_files): ++ # This directory contains a subdirectory with sequence files ++ # Treat the parent as a sequence directory ++ yield path ++ break ++ except (OSError, PermissionError): ++ # Skip if we can't access the subdirectory ++ continue + + +``` + +
+ + + +--- + + + +--- + + +Reverting that change and adding debug logging to see why it's being skipped: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 11, lines removed: 18 + +```diff +@@ -477,30 +477,23 @@ + + +- def find_sequence_dirs(root: Path) -> Iterator[Path]: ++ def find_sequence_dirs(root: Path, *, verbose: bool = False) -> Iterator[Path]: + for dirpath, dirnames, filenames in os.walk(root): + path = Path(dirpath) + dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] + if is_archive_path(path): ++ if verbose: ++ rel = path.relative_to(root) if path.is_relative_to(root) else path ++ log("scan", f"Skipping archive path: {rel}", verbose_only=True, verbose=verbose) + continue + has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames) + if has_frames: + yield path +- # Also check subdirectories for sequence files (in case the directory itself has no files) +- # but only if the directory is not too deep (to avoid scanning archive directories) +- elif not any(part in ("_archive", "_CURRENT", "_zipped") for part in path.parts): +- # Check if any immediate subdirectory has sequence files +- for subdir in dirnames: +- subdir_path = path / subdir +- if subdir_path.is_dir(): +- try: +- subdir_files = list(subdir_path.iterdir()) +- if any(f.is_file() and f.suffix.lower() in SEQUENCE_EXTENSIONS for f in subdir_files): +- # This directory contains a subdirectory with sequence files +- # Treat the parent as a sequence directory +- yield path +- break +- except (OSError, PermissionError): +- # Skip if we can't access the subdirectory +- continue ++ elif verbose: ++ # Log directories that don't have sequence files for debugging ++ rel = path.relative_to(root) if path.is_relative_to(root) else path ++ if "scab" in path.name.lower() or "v2" in path.name.lower(): ++ # Special logging for directories that might be sequences ++ frame_extensions = [f for f in filenames if Path(f).suffix.lower() in SEQUENCE_EXTENSIONS] ++ log("scan", f"Directory {rel} has no sequence files (found {len(filenames)} files, {len(frame_extensions)} with sequence extensions)", verbose_only=True, verbose=verbose) + + +``` + +
+ + + +--- + + + +--- + + +Updating the call to `find_sequence_dirs` to pass the verbose flag: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 1, lines removed: 1 + +```diff +@@ -807,5 +807,5 @@ + + if RENDER_ROOT.exists(): +- for seq_dir in find_sequence_dirs(RENDER_ROOT): ++ for seq_dir in find_sequence_dirs(RENDER_ROOT, verbose=verbose): + total_scanned += 1 + rel = seq_dir.relative_to(RENDER_ROOT) +``` + +
+ + + +--- + + +
+ Tool use: **read_lints** • Read lints for 1 file + +Lint paths: + +- `zip_sequences.py` + +Lint results + + +**No lint errors found** +
+ + +--- + + + +--- + + +Improving detection to handle sequences where images are in subdirectories. Updating the logic: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 1, lines removed: undefined + +```diff +@@ -478,4 +478,5 @@ + + def find_sequence_dirs(root: Path, *, verbose: bool = False) -> Iterator[Path]: ++ seen_dirs = set() # Track directories we've already yielded to avoid duplicates + for dirpath, dirnames, filenames in os.walk(root): + path = Path(dirpath) +``` + +**Chunk 2** +Lines added: 19, lines removed: 3 + +```diff +@@ -486,14 +487,30 @@ + log("scan", f"Skipping archive path: {rel}", verbose_only=True, verbose=verbose) + continue ++ ++ # Check if this directory has sequence files directly + has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames) + if has_frames: +- yield path ++ path_resolved = path.resolve() ++ if path_resolved not in seen_dirs: ++ seen_dirs.add(path_resolved) ++ yield path + elif verbose: + # Log directories that don't have sequence files for debugging + rel = path.relative_to(root) if path.is_relative_to(root) else path +- if "scab" in path.name.lower() or "v2" in path.name.lower(): ++ if "scab" in path.name.lower(): + # Special logging for directories that might be sequences + frame_extensions = [f for f in filenames if Path(f).suffix.lower() in SEQUENCE_EXTENSIONS] +- log("scan", f"Directory {rel} has no sequence files (found {len(filenames)} files, {len(frame_extensions)} with sequence extensions)", verbose_only=True, verbose=verbose) ++ all_files = list(path.iterdir()) if path.exists() else [] ++ log("scan", f"Directory {rel}: {len(filenames)} files in dir, {len(frame_extensions)} with sequence extensions, {len(all_files)} total items", verbose_only=True, verbose=verbose) ++ # Check subdirectories ++ for subdir in dirnames[:5]: # Check first 5 subdirs ++ subdir_path = path / subdir ++ try: ++ subdir_files = list(subdir_path.iterdir()) if subdir_path.exists() else [] ++ subdir_frame_files = [f for f in subdir_files if f.is_file() and f.suffix.lower() in SEQUENCE_EXTENSIONS] ++ if subdir_frame_files: ++ log("scan", f" Subdirectory {subdir} has {len(subdir_frame_files)} sequence files", verbose_only=True, verbose=verbose) ++ except (OSError, PermissionError): ++ pass + + +``` + +
+ + + +--- + + + +--- + + +Run `ZipSeqArchv.bat` with `--verbose` to see why `scab_v2.1` is being skipped. The logging will show: +1. Whether the directory is being scanned +2. How many files it contains +3. Whether any subdirectories have sequence files + +The current logic only detects directories that have sequence files directly in them. If `scab_v2.1` has images only in subdirectories, those subdirectories are detected as separate sequences, not `scab_v2.1` itself. + +If you want `scab_v2.1` to be treated as a single sequence even when images are in subdirectories, we can update the detection logic. Share the verbose output and we can adjust accordingly. + +--- + +_**User (2025-11-25 21:04Z)**_ + +PS P:\251119_Tatt2Away_How-it-Works\Renders> P:\251119_Tatt2Away_How-it-Works\Renders\ZipSeqArchv.bat + +[init] Loading configuration sources... + +[init] Reading project config at P:\251119_Tatt2Away_How-it-Works\.config\config.json + +[init] Configuration loaded from project + +[init] zip_sequences starting up... + +[init] Working directory: P:\251119_Tatt2Away_How-it-Works + +[init] Mode: zip, zipper: 7z, jobs arg: auto + +[init] 7z executable: C:\ProgramData\chocolatey\bin\7z.EXE, Max7zInst: auto, method: LZMA2, level: 5 + +[init] Scanning sequences under \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders + +[scan] [1] Inspecting 231016_audio_skin-top_2_CH + +[scan] Skipping 231016_audio_skin-top_2_CH (unchanged since archive) + +[scan] [2] Inspecting scab_v2.1 + +[scan] Queued scab_v2.1 for compression (~1.31GB) [1 total] + +[scan] [3] Inspecting tatt2away_process_v3 + +[scan] Queued tatt2away_process_v3 for compression (~0.67GB) [2 total] + +[scan] [4] Inspecting Tatt2Away_stages_v2_CM + +[scan] Skipping Tatt2Away_stages_v2_CM (unchanged since archive) + +[zip] RAM: 102.2GB available (80% of total), 97.1GB for compression (95%) + +[zip] Estimated RAM per sequence: 6.0GB total across 2 sequences + +[zip] All sequences fit in RAM → using 2 workers (one per sequence) + +[zip] Final worker count: 2 (requested: None) + +[init] Preparing to compress 2 sequence(s) with 2 worker(s) + +[zip] Renders\scab_v2.1 -> Renders\_zipped\scab_v2.7z + +[zip] Renders\tatt2away_process_v3 -> Renders\_zipped\tatt2away_process_v3.7z + +[zip] 7z command: C:\ProgramData\chocolatey\bin\7z.EXE a -y -bb0 -mx=5 -t7z -m0=LZMA2 -md=1024m -mmt=on \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped\tmpaxb_br7j.7z @C:\Users\Nathan\AppData\Local\Temp\tmpsim_da0y.lst + +[zip] 7z command: C:\ProgramData\chocolatey\bin\7z.EXE a -y -bb0 -mx=5 -t7z -m0=LZMA2 -md=1024m -mmt=on \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped\tmp2vv7izgy.7z @C:\Users\Nathan\AppData\Local\Temp\tmpv25oxn65.lst + +[zip] 1/2 tatt2away_process_v3 + +[zip] 2/2 scab_v2.1 + +[zip] Updated 2 sequence archive(s). + +[zip] Archives updated. Stage manually with `git add Renders/_zipped`, if desired. + +[scan] Summary: scanned 4, quick-skipped 2, state-skipped 0, empty 0, queued 2 + +[zip] Scanning for orphan archives in \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped + +[zip] Archive 231016_audio_skin-top_2_CH.7z has matching sequence directory; keeping + +[zip] Removing orphan archive scab_v2.7z + +[zip] Deleted archive: scab_v2.7z + +[zip] Removed orphan metadata scab_v2.7z.meta.json + +[zip] Archive tatt2away_process_v3.7z has matching sequence directory; keeping + +[zip] Archive Tatt2Away_stages_v2_CM.7z has matching sequence directory; keeping + +[zip] Removed 1 orphan archive(s). + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +Archive path is incorrect: `scab_v2.1` is being archived as `scab_v2.7z` instead of `scab_v2.1.7z`, so the `.1` is dropped. Checking `archive_path_for`: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +
+Tool use: **grep** • Grep for "def archive_path_for" in "zip_sequences.py" • 1 match + +Output mode: content + + +| File | Content | Line | +|------|------|------| +| `zip_sequences.py` | `def archive_path_for(seq_dir: Path) -> Path:` | L566 | + + +
+ + +--- + + + + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + + + +--- + + + +--- + + +The issue is that `archive_path_for` uses `.with_suffix()`, which replaces the suffix. For `scab_v2.1`, `.1` is treated as a suffix and replaced with `.7z`, producing `scab_v2.7z` instead of `scab_v2.1.7z`. Fixing it to append the suffix instead: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 2, lines removed: 1 + +```diff +@@ -567,5 +567,6 @@ + rel = seq_dir.relative_to(RENDER_ROOT) + suffix = ".7z" if ZIPPER_TYPE == "7z" else ".zip" +- return (ARCHIVE_ROOT / rel).with_suffix(suffix) ++ # Append suffix instead of replacing, since directory names might have dots (e.g., "scab_v2.1") ++ return ARCHIVE_ROOT / f"{rel}{suffix}" + + +``` + +
+ + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +Fixing `sequence_dir_for` to handle this correctly: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 8, lines removed: 1 + +```diff +@@ -573,5 +573,12 @@ + def sequence_dir_for(zip_path: Path) -> Path: + rel = zip_path.relative_to(ARCHIVE_ROOT) +- return (RENDER_ROOT / rel).with_suffix("") ++ # Remove the archive suffix (.7z or .zip) from the end ++ # Handle both .7z and .zip extensions ++ rel_str = str(rel) ++ if rel_str.endswith(".7z"): ++ rel_str = rel_str[:-3] ++ elif rel_str.endswith(".zip"): ++ rel_str = rel_str[:-4] ++ return RENDER_ROOT / rel_str + + +``` + +
+ + + +--- + + +
+ Tool use: **read_lints** • Read lints for 1 file + +Lint paths: + +- `zip_sequences.py` + +Lint results + + +**No lint errors found** +
+ + +--- + + + +--- + + +Fixed the issue: `archive_path_for` used `.with_suffix()`, which replaced the suffix. For `scab_v2.1`, `.1` was treated as a suffix and replaced with `.7z`, producing `scab_v2.7z` instead of `scab_v2.1.7z`. + +**Changes made:** +1. **`archive_path_for`**: Appends the suffix instead of replacing it, so `scab_v2.1` becomes `scab_v2.1.7z` +2. **`sequence_dir_for`**: Removes the archive extension (`.7z` or `.zip`) from the end to correctly map back to the sequence directory + +When you run `ZipSeqArchv.bat` again: +- `scab_v2.1` will be archived as `scab_v2.1.7z` (not `scab_v2.7z`) +- The cleanup function will correctly match the archive to the sequence directory +- It won't be removed as an orphan + +This also fixes the issue for any other directory names containing dots (e.g., `v2.1`, `test.1.2`, etc.). + +--- + +_**User (2025-11-25 21:04Z)**_ + +is the compression lvl 0 working on this? it doesn't seem to be going very fast + +PS P:\251119_Tatt2Away_How-it-Works\Renders> P:\251119_Tatt2Away_How-it-Works\Renders\ZipSeqArchv.bat + +[init] Loading configuration sources... + +[init] Reading project config at P:\251119_Tatt2Away_How-it-Works\.config\config.json + +[init] Configuration loaded from project + +[init] zip_sequences starting up... + +[init] Working directory: P:\251119_Tatt2Away_How-it-Works + +[init] Mode: zip, zipper: 7z, jobs arg: auto + +[init] 7z executable: C:\ProgramData\chocolatey\bin\7z.EXE, Max7zInst: auto, method: LZMA2, level: 0 + +[init] Scanning sequences under \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders + +[scan] [1] Inspecting 231016_audio_skin-top_2_CH + +[scan] Skipping 231016_audio_skin-top_2_CH (unchanged since archive) + +[scan] [2] Inspecting scab_v2.1 + +[scan] Queued scab_v2.1 for compression (~1.31GB) [1 total] + +[scan] [3] Inspecting tatt2away_process_v3 + +[scan] Skipping tatt2away_process_v3 (unchanged since archive) + +[scan] [4] Inspecting Tatt2Away_stages_v2_CM + +[scan] Skipping Tatt2Away_stages_v2_CM (unchanged since archive) + +[zip] RAM: 102.2GB available (80% of total), 97.1GB for compression (95%) + +[zip] Estimated RAM per sequence: 3.0GB total across 1 sequences + +[zip] All sequences fit in RAM → using 1 workers (one per sequence) + +[zip] Final worker count: 1 (requested: None) + +[init] Preparing to compress 1 sequence(s) with 1 worker(s) + +[zip] Renders\scab_v2.1 -> Renders\_zipped\scab_v2.1.7z + +[zip] 7z command: C:\ProgramData\chocolatey\bin\7z.EXE a -y -bb0 -mx=0 -t7z -m0=LZMA2 -md=1024m -mmt=32 \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped\tmpq3l68ie4.7z @C:\Users\Nathan\AppData\Local\Temp\tmp57ak8mgp.lst + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +At compression level 0, the script still uses `-m0=LZMA2`, which compresses. At level 0, we should use `-m0=Copy` (store). Updating the code to use Copy when level is 0: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + +
+Tool use: **grep** • Grep for "COMPRESSION_LEVEL" in "zip_sequences.py" • 11 matches + +Output mode: content + + +| File | Content | Line | +|------|------|------| +| `zip_sequences.py` | `COMPRESSION_LEVEL = CONFIG.get("compression", 9)` | L114 | +| `zip_sequences.py` | `if isinstance(COMPRESSION_LEVEL, str):` | L115 | +| `zip_sequences.py` | `COMPRESSION_LEVEL = int(COMPRESSION_LEVEL)` | L117 | +| `zip_sequences.py` | `COMPRESSION_LEVEL = 9` | L119 | +| `zip_sequences.py` | `if not isinstance(COMPRESSION_LEVEL, int):` | L120 | +| `zip_sequences.py` | `COMPRESSION_LEVEL = 9` | L121 | +| `zip_sequences.py` | `COMPRESSION_LEVEL = max(0, min(9, COMPRESSION_LEVEL))` | L122 | +| `zip_sequences.py` | `f"-mx=\{COMPRESSION_LEVEL\}",` | L645 | +| `zip_sequences.py` | `if COMPRESSION_LEVEL <= 0:` | L743 | +| `zip_sequences.py` | `zip_kwargs = \{"compresslevel": COMPRESSION_LEVEL\}` | L748 | +| `zip_sequences.py` | `log("init", f"7z executable: \{exe\}, Max7zInst: \{max_inst\}, method: \{COMPRESSION_METHOD\}, level: \{COMPRESSION_LEVEL\}")` | L1117 | + + +
+ + +--- + + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + + + +--- + + + +--- + + +Updating the code to use Copy (store) when compression level is 0: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 42, lines removed: 35 + +```diff +@@ -648,40 +648,47 @@ + + # Set compression method and memory/dictionary size based on method +- # PPMd uses memory parameter in method string, LZMA2 uses -md flag ++ # At compression level 0, use Copy (store) method for maximum speed + FIXED_DICT_SIZE_MB = 1024 +- if COMPRESSION_METHOD == "PPMd": +- # PPMd: specify memory as part of method string +- cmd.append(f"-m0=PPMd:mem={FIXED_DICT_SIZE_MB}m") +- elif COMPRESSION_METHOD == "LZMA2": +- # LZMA2: use -md for dictionary size +- cmd.append("-m0=LZMA2") +- cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") +- elif COMPRESSION_METHOD == "BZip2": +- # BZip2: use -md for dictionary size (smaller max: 900KB) +- max_bzip2_dict = min(FIXED_DICT_SIZE_MB, 900) # BZip2 max is 900KB +- cmd.append("-m0=BZip2") +- cmd.append(f"-md={max_bzip2_dict}k") +- elif COMPRESSION_METHOD == "Deflate": +- # Deflate: doesn't use dictionary size parameter +- cmd.append("-m0=Deflate") +- else: +- # Fallback: use LZMA2 +- cmd.append("-m0=LZMA2") +- cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") +- +- # CPU thread allocation: when there's only 1 worker, use all CPU cores +- # When there are multiple workers, use auto mode to let 7z decide +- # Note: PPMd is single-threaded and won't benefit from -mmt +- cpu_cores = os.cpu_count() or 1 +- if COMPRESSION_METHOD == "PPMd": +- # PPMd is single-threaded, so -mmt won't help +- # But we can still set it for consistency +- cmd.append("-mmt=on") +- elif worker_count == 1: +- # Single worker: use all CPU cores for maximum speed (LZMA2, BZip2, Deflate support this) +- cmd.append(f"-mmt={cpu_cores}") +- else: +- # Multiple workers: use auto mode (7z will manage threads) +- cmd.append("-mmt=on") ++ if COMPRESSION_LEVEL == 0: ++ # Level 0 = no compression, just store files (fastest) ++ cmd.append("-m0=Copy") ++ # Copy method doesn't need threading, but enable it anyway for consistency ++ cmd.append("-mmt=on") ++ else: ++ # Compression levels 1-9: use configured compression method ++ if COMPRESSION_METHOD == "PPMd": ++ # PPMd: specify memory as part of method string ++ cmd.append(f"-m0=PPMd:mem={FIXED_DICT_SIZE_MB}m") ++ elif COMPRESSION_METHOD == "LZMA2": ++ # LZMA2: use -md for dictionary size ++ cmd.append("-m0=LZMA2") ++ cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") ++ elif COMPRESSION_METHOD == "BZip2": ++ # BZip2: use -md for dictionary size (smaller max: 900KB) ++ max_bzip2_dict = min(FIXED_DICT_SIZE_MB, 900) # BZip2 max is 900KB ++ cmd.append("-m0=BZip2") ++ cmd.append(f"-md={max_bzip2_dict}k") ++ elif COMPRESSION_METHOD == "Deflate": ++ # Deflate: doesn't use dictionary size parameter ++ cmd.append("-m0=Deflate") ++ else: ++ # Fallback: use LZMA2 ++ cmd.append("-m0=LZMA2") ++ cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") ++ ++ # CPU thread allocation: when there's only 1 worker, use all CPU cores ++ # When there are multiple workers, use auto mode to let 7z decide ++ # Note: PPMd is single-threaded and won't benefit from -mmt ++ cpu_cores = os.cpu_count() or 1 ++ if COMPRESSION_METHOD == "PPMd": ++ # PPMd is single-threaded, so -mmt won't help ++ # But we can still set it for consistency ++ cmd.append("-mmt=on") ++ elif worker_count == 1: ++ # Single worker: use all CPU cores for maximum speed (LZMA2, BZip2, Deflate support this) ++ cmd.append(f"-mmt={cpu_cores}") ++ else: ++ # Multiple workers: use auto mode (7z will manage threads) ++ cmd.append("-mmt=on") + + cmd.extend([ +``` + +
+ + + +--- + + +
+ Tool use: **read_lints** • Read lints for 1 file + +Lint paths: + +- `zip_sequences.py` + +Lint results + + +**No lint errors found** +
+ + +--- + + + +--- + + +Fixed: at compression level 0, the script was still using LZMA2, which compresses. At level 0, it now uses `-m0=Copy` (store), which packages files without compression. + +**What changed:** +- When `compression: 0` is set, the script now uses `-m0=Copy` instead of the configured compression method +- Copy mode stores files without compression, so it's limited mainly by disk I/O speed +- Compression levels 1-9 continue to use the configured method (LZMA2, PPMd, etc.) + +When you run `ZipSeqArchv.bat` with `compression: 0`, it should be much faster since it's just copying files into the archive without compression. The 7z command will now show `-m0=Copy` instead of `-m0=LZMA2` when compression level is 0. + +--- + +_**User (2025-11-25 21:04Z)**_ + +is good? + +PS P:\251119_Tatt2Away_How-it-Works\Renders> P:\251119_Tatt2Away_How-it-Works\Renders\ZipSeqArchv.bat + +[init] Loading configuration sources... + +[init] Reading project config at P:\251119_Tatt2Away_How-it-Works\.config\config.json + +[init] Configuration loaded from project + +[init] zip_sequences starting up... + +[init] Working directory: P:\251119_Tatt2Away_How-it-Works + +[init] Mode: zip, zipper: 7z, jobs arg: auto + +[init] 7z executable: C:\ProgramData\chocolatey\bin\7z.EXE, Max7zInst: auto, method: LZMA2, level: 0 + +[init] Scanning sequences under \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders + +[scan] [1] Inspecting 231016_audio_skin-top_2_CH + +[scan] Skipping 231016_audio_skin-top_2_CH (unchanged since archive) + +[scan] [2] Inspecting scab_v2.1 + +[scan] Queued scab_v2.1 for compression (~1.31GB) [1 total] + +[scan] [3] Inspecting tatt2away_process_v3 + +[scan] Skipping tatt2away_process_v3 (unchanged since archive) + +[scan] [4] Inspecting Tatt2Away_stages_v2_CM + +[scan] Skipping Tatt2Away_stages_v2_CM (unchanged since archive) + +[zip] RAM: 102.2GB available (80% of total), 97.1GB for compression (95%) + +[zip] Estimated RAM per sequence: 3.0GB total across 1 sequences + +[zip] All sequences fit in RAM → using 1 workers (one per sequence) + +[zip] Final worker count: 1 (requested: None) + +[init] Preparing to compress 1 sequence(s) with 1 worker(s) + +[zip] Renders\scab_v2.1 -> Renders\_zipped\scab_v2.1.7z + +[zip] 7z command: C:\ProgramData\chocolatey\bin\7z.EXE a -y -bb0 -mx=0 -t7z -m0=Copy -mmt=on \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped\tmpglnjsei8.7z @C:\Users\Nathan\AppData\Local\Temp\tmp_vsbg9m6.lst + +[zip] 1/1 scab_v2.1 + +[zip] Updated 1 sequence archive(s). + +[zip] Archives updated. Stage manually with `git add Renders/_zipped`, if desired. + +[scan] Summary: scanned 4, quick-skipped 3, state-skipped 0, empty 0, queued 1 + +[zip] Scanning for orphan archives in \\NEXUS\proj\251119_Tatt2Away_How-it-Works\Renders\_zipped + +[zip] Archive 231016_audio_skin-top_2_CH.7z has matching sequence directory; keeping + +[zip] Archive scab_v2.1.7z has matching sequence directory; keeping + +[zip] Archive tatt2away_process_v3.7z has matching sequence directory; keeping + +[zip] Archive Tatt2Away_stages_v2_CM.7z has matching sequence directory; keeping + +[zip] Removing orphan archive tmp4vygazhw.7z + +[zip] Deleted archive: tmp4vygazhw.7z + +[zip] Removing orphan archive tmp8caqu_bp.7z + +[zip] Deleted archive: tmp8caqu_bp.7z + +[zip] Removing orphan archive tmp8nuwz4x1.7z + +[zip] Deleted archive: tmp8nuwz4x1.7z + +[zip] Removing orphan archive tmpfs0howr_.7z + +[zip] Deleted archive: tmpfs0howr_.7z + +[zip] Removed 4 orphan archive(s). + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + +Yes, it's working correctly. + +1. Compression level 0 is working: the 7z command shows `-m0=Copy` instead of `-m0=LZMA2`, so it's using Copy (store) with no compression, which should be faster. + +2. Archive path is correct: `scab_v2.1.7z` is created correctly (not `scab_v2.7z`). + +3. Orphan cleanup is working: it found and removed 4 leftover temporary files (`tmp*.7z`) that should have been cleaned up. + +Everything looks good. With compression level 0, archiving should be much faster since it's just copying files into the archive without compression. + +--- + diff --git a/config.json b/config.json index 3c87098..20eb7fd 100644 --- a/config.json +++ b/config.json @@ -2,7 +2,7 @@ "dailyFormat": "daily_YYMMDD", "structDir": "A:\\1 Amazon_Active_Projects\\3 ProjectStructure", "zipper": "7z", - "compression": 5, + "compression": 0, "Max7zInst": 0 } diff --git a/zip_sequences.py b/zip_sequences.py index 8f0f8b9..56aecda 100644 --- a/zip_sequences.py +++ b/zip_sequences.py @@ -476,15 +476,42 @@ def is_archive_path(path: Path) -> bool: return any(part in ("_archive", "_CURRENT") for part in path.parts) -def find_sequence_dirs(root: Path) -> Iterator[Path]: +def find_sequence_dirs(root: Path, *, verbose: bool = False) -> Iterator[Path]: + seen_dirs = set() # Track directories we've already yielded to avoid duplicates for dirpath, dirnames, filenames in os.walk(root): path = Path(dirpath) dirnames[:] = [d for d in dirnames if d not in ("_archive", "_CURRENT")] if is_archive_path(path): + if verbose: + rel = path.relative_to(root) if path.is_relative_to(root) else path + log("scan", f"Skipping archive path: {rel}", verbose_only=True, verbose=verbose) continue + + # Check if this directory has sequence files directly has_frames = any(Path(dirpath, f).suffix.lower() in SEQUENCE_EXTENSIONS for f in filenames) if has_frames: - yield path + path_resolved = path.resolve() + if path_resolved not in seen_dirs: + seen_dirs.add(path_resolved) + yield path + elif verbose: + # Log directories that don't have sequence files for debugging + rel = path.relative_to(root) if path.is_relative_to(root) else path + if "scab" in path.name.lower(): + # Special logging for directories that might be sequences + frame_extensions = [f for f in filenames if Path(f).suffix.lower() in SEQUENCE_EXTENSIONS] + all_files = list(path.iterdir()) if path.exists() else [] + log("scan", f"Directory {rel}: {len(filenames)} files in dir, {len(frame_extensions)} with sequence extensions, {len(all_files)} total items", verbose_only=True, verbose=verbose) + # Check subdirectories + for subdir in dirnames[:5]: # Check first 5 subdirs + subdir_path = path / subdir + try: + subdir_files = list(subdir_path.iterdir()) if subdir_path.exists() else [] + subdir_frame_files = [f for f in subdir_files if f.is_file() and f.suffix.lower() in SEQUENCE_EXTENSIONS] + if subdir_frame_files: + log("scan", f" Subdirectory {subdir} has {len(subdir_frame_files)} sequence files", verbose_only=True, verbose=verbose) + except (OSError, PermissionError): + pass def iter_sequence_files(seq_dir: Path) -> Iterator[Path]: @@ -539,12 +566,20 @@ def state_changed(seq_state: dict, stored_state: dict | None) -> bool: def archive_path_for(seq_dir: Path) -> Path: rel = seq_dir.relative_to(RENDER_ROOT) suffix = ".7z" if ZIPPER_TYPE == "7z" else ".zip" - return (ARCHIVE_ROOT / rel).with_suffix(suffix) + # Append suffix instead of replacing, since directory names might have dots (e.g., "scab_v2.1") + return ARCHIVE_ROOT / f"{rel}{suffix}" def sequence_dir_for(zip_path: Path) -> Path: rel = zip_path.relative_to(ARCHIVE_ROOT) - return (RENDER_ROOT / rel).with_suffix("") + # Remove the archive suffix (.7z or .zip) from the end + # Handle both .7z and .zip extensions + rel_str = str(rel) + if rel_str.endswith(".7z"): + rel_str = rel_str[:-3] + elif rel_str.endswith(".zip"): + rel_str = rel_str[:-4] + return RENDER_ROOT / rel_str def state_path_for(zip_path: Path) -> Path: @@ -612,42 +647,49 @@ def zip_sequence(seq_dir: Path, zip_path: Path, per_job_memory_limit: int | None ] # Set compression method and memory/dictionary size based on method - # PPMd uses memory parameter in method string, LZMA2 uses -md flag + # At compression level 0, use Copy (store) method for maximum speed FIXED_DICT_SIZE_MB = 1024 - if COMPRESSION_METHOD == "PPMd": - # PPMd: specify memory as part of method string - cmd.append(f"-m0=PPMd:mem={FIXED_DICT_SIZE_MB}m") - elif COMPRESSION_METHOD == "LZMA2": - # LZMA2: use -md for dictionary size - cmd.append("-m0=LZMA2") - cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") - elif COMPRESSION_METHOD == "BZip2": - # BZip2: use -md for dictionary size (smaller max: 900KB) - max_bzip2_dict = min(FIXED_DICT_SIZE_MB, 900) # BZip2 max is 900KB - cmd.append("-m0=BZip2") - cmd.append(f"-md={max_bzip2_dict}k") - elif COMPRESSION_METHOD == "Deflate": - # Deflate: doesn't use dictionary size parameter - cmd.append("-m0=Deflate") - else: - # Fallback: use LZMA2 - cmd.append("-m0=LZMA2") - cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") - - # CPU thread allocation: when there's only 1 worker, use all CPU cores - # When there are multiple workers, use auto mode to let 7z decide - # Note: PPMd is single-threaded and won't benefit from -mmt - cpu_cores = os.cpu_count() or 1 - if COMPRESSION_METHOD == "PPMd": - # PPMd is single-threaded, so -mmt won't help - # But we can still set it for consistency + if COMPRESSION_LEVEL == 0: + # Level 0 = no compression, just store files (fastest) + cmd.append("-m0=Copy") + # Copy method doesn't need threading, but enable it anyway for consistency cmd.append("-mmt=on") - elif worker_count == 1: - # Single worker: use all CPU cores for maximum speed (LZMA2, BZip2, Deflate support this) - cmd.append(f"-mmt={cpu_cores}") else: - # Multiple workers: use auto mode (7z will manage threads) - cmd.append("-mmt=on") + # Compression levels 1-9: use configured compression method + if COMPRESSION_METHOD == "PPMd": + # PPMd: specify memory as part of method string + cmd.append(f"-m0=PPMd:mem={FIXED_DICT_SIZE_MB}m") + elif COMPRESSION_METHOD == "LZMA2": + # LZMA2: use -md for dictionary size + cmd.append("-m0=LZMA2") + cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") + elif COMPRESSION_METHOD == "BZip2": + # BZip2: use -md for dictionary size (smaller max: 900KB) + max_bzip2_dict = min(FIXED_DICT_SIZE_MB, 900) # BZip2 max is 900KB + cmd.append("-m0=BZip2") + cmd.append(f"-md={max_bzip2_dict}k") + elif COMPRESSION_METHOD == "Deflate": + # Deflate: doesn't use dictionary size parameter + cmd.append("-m0=Deflate") + else: + # Fallback: use LZMA2 + cmd.append("-m0=LZMA2") + cmd.append(f"-md={FIXED_DICT_SIZE_MB}m") + + # CPU thread allocation: when there's only 1 worker, use all CPU cores + # When there are multiple workers, use auto mode to let 7z decide + # Note: PPMd is single-threaded and won't benefit from -mmt + cpu_cores = os.cpu_count() or 1 + if COMPRESSION_METHOD == "PPMd": + # PPMd is single-threaded, so -mmt won't help + # But we can still set it for consistency + cmd.append("-mmt=on") + elif worker_count == 1: + # Single worker: use all CPU cores for maximum speed (LZMA2, BZip2, Deflate support this) + cmd.append(f"-mmt={cpu_cores}") + else: + # Multiple workers: use auto mode (7z will manage threads) + cmd.append("-mmt=on") cmd.extend([ str(temp_zip_abs), @@ -796,7 +838,7 @@ def run_zip(requested_workers: int | None, *, verbose: bool) -> int: queued = 0 if RENDER_ROOT.exists(): - for seq_dir in find_sequence_dirs(RENDER_ROOT): + for seq_dir in find_sequence_dirs(RENDER_ROOT, verbose=verbose): total_scanned += 1 rel = seq_dir.relative_to(RENDER_ROOT) if total_scanned <= 5 or total_scanned % 10 == 0: