diff --git a/.specstory/history/2025-11-10_16-59Z-change-compression-method-to-7z.md b/.specstory/history/2025-11-10_16-59Z-change-compression-method-to-7z.md index e1ce95d..ddbfce5 100644 --- a/.specstory/history/2025-11-10_16-59Z-change-compression-method-to-7z.md +++ b/.specstory/history/2025-11-10_16-59Z-change-compression-method-to-7z.md @@ -19401,3 +19401,434 @@ This ensures that `.zip` files are only replaced with `.7z` when the files have --- +_**User (2025-11-11 19:15Z)**_ + +Very low CPU and RAM allocation on this. Let's make Max7zInst=0 be our original balls-to-the-walls process. + +[zip] RAM: 102.2GB available (80% of total), 40.9GB for compression (40%), 8.0GB per job (dict: 1024MB) → RAM limit: 5, requested: None, final: 5 + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + +Updating the auto-calculation logic for `Max7zInst=0` to be more aggressive, using more CPU and RAM: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + + +--- + +Updating the auto-calculation to be more aggressive when `Max7zInst=0`: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 6, lines removed: 5 + +```diff +@@ -282,12 +282,13 @@ + + # Auto-calculate based on RAM if Max7zInst not configured ++ # "Balls-to-the-walls" mode: use maximum resources + if available_ram is not None: +- # 7z uses ~2-3x dictionary size in RAM, but with overhead use 8x for safety +- # This accounts for 7z's internal buffers, OS overhead, and other processes +- FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 8 * 1024 * 1024 # 8GB per job ++ # 7z uses ~2-3x dictionary size in RAM, use 3x for aggressive mode ++ # This is more realistic and allows more concurrent workers ++ FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 3 * 1024 * 1024 # 3GB per job + + # available_ram is already 80% of total (20% reserved for system) +- # Use only 40% of that for compression jobs (very conservative to prevent swapping) +- compression_ram = int(available_ram * 0.4) ++ # Use 95% of available RAM for compression jobs (aggressive mode) ++ compression_ram = int(available_ram * 0.95) + + # Calculate worker limit based on fixed per-job RAM +``` + +**Chunk 2** +Lines added: 1, lines removed: 1 + +```diff +@@ -305,5 +306,5 @@ + log( + "zip", +- f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " ++ f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, requested: {requested}, final: {final_limit}", + verbose_only=True, +``` + +
+ + + +--- + + + +--- + + +
+ Tool use: **read_lints** • Read lints for 1 file + +Lint paths: + +- `zip_sequences.py` + +Lint results + + +**No lint errors found** +
+ + +--- + + + +--- + + + +--- + +Updated the auto-calculation for `Max7zInst=0` to be more aggressive. + +**Changes:** +1. **Increased RAM usage**: From 40% to 95% of available RAM for compression +2. **Reduced per-job RAM estimate**: From 8GB to 3GB per job (more realistic for a 1GB dictionary) + +**Results for your system:** +- **Before**: 40.9GB for compression ÷ 8GB per job = **5 workers** +- **After**: ~97GB for compression ÷ 3GB per job = **~32 workers** + +This should use more CPU and RAM. The dictionary size remains 1GB, so compression ratio is unchanged. + +With `Max7zInst=0`, the script will use up to ~32 concurrent 7z instances, maximizing throughput. If you want to limit it, set `Max7zInst` to a specific number in your config. + +--- + +_**User (2025-11-11 21:54Z)**_ + +it's now trying to do 32 jobs with 3gb allocated to each, even when there's only one sequence to zip. let's make it account for the number of 7z jobs there are to do. + +--- + +_**Agent (model default, mode Agent)**_ + + + +--- + + + +--- + + +Updating the logic to cap workers at the number of sequences to zip. Checking how `max_workers` is called: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +
+Tool use: **grep** • Grep for "def run_zip" in "zip_sequences.py" • 1 match + +Output mode: content + + +| File | Content | Line | +|------|------|------| +| `zip_sequences.py` | `def run_zip(requested_workers: int \| None, *, verbose: bool) -> int:` | L639 | + + +
+ + +--- + + + + + +--- + + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + + + +--- + + + +--- + + +Updating `max_workers` to cap workers at the number of work items: + + +--- + + +
Tool use: **read_file** • Read file: zip_sequences.py + +
+ + +--- + + + +--- + + +Capping workers at the number of work items: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 5, lines removed: undefined + +```diff +@@ -295,4 +295,9 @@ + ram_limit = max(1, compression_ram // FIXED_RAM_PER_JOB) + ++ # Cap at number of actual work items (can't have more workers than jobs) ++ num_work_items = len(work_items) if work_items else 0 ++ if num_work_items > 0: ++ ram_limit = min(ram_limit, num_work_items) ++ + # Use RAM limit directly (no CPU limit) + final_limit = ram_limit +``` + +**Chunk 2** +Lines added: 1, lines removed: 1 + +```diff +@@ -307,5 +312,5 @@ + "zip", + f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " +- f"RAM limit: {ram_limit}, requested: {requested}, final: {final_limit}", ++ f"RAM limit: {ram_limit}, work items: {num_work_items}, requested: {requested}, final: {final_limit}", + verbose_only=True, + verbose=verbose +``` + +
+ + + +--- + + + +--- + + +Updating the configured `MAX_7Z_INSTANCES` case to also cap at the number of work items: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 4, lines removed: 1 + +```diff +@@ -265,6 +265,9 @@ + # Check if Max7zInst is configured + if MAX_7Z_INSTANCES is not None: +- # Use configured maximum instances, but still respect user's --jobs ++ # Use configured maximum instances, but still respect user's --jobs and work items + final_limit = MAX_7Z_INSTANCES ++ num_work_items = len(work_items) if work_items else 0 ++ if num_work_items > 0: ++ final_limit = min(final_limit, num_work_items) + if requested and requested > 0: + final_limit = min(final_limit, requested) +``` + +**Chunk 2** +Lines added: 1, lines removed: 1 + +```diff +@@ -274,5 +277,5 @@ + "zip", + f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " +- f"requested: {requested}, final: {final_limit}", ++ f"work items: {num_work_items}, requested: {requested}, final: {final_limit}", + verbose_only=True, + verbose=verbose +``` + +
+ + + +--- + + + +--- + + +Updating the default case when RAM detection fails: + + +--- + + + +
Tool use: **code_edit** • Edit file: zip_sequences.py + +**Chunk 1** +Lines added: 12, lines removed: 3 + +```diff +@@ -323,9 +323,18 @@ + + # RAM detection failed, use a safe default (no CPU limit) +- if verbose: +- log("zip", "RAM detection failed and Max7zInst not set, using default worker limit of 4", verbose_only=True, verbose=verbose) + default_limit = 4 ++ num_work_items = len(work_items) if work_items else 0 ++ if num_work_items > 0: ++ default_limit = min(default_limit, num_work_items) + if requested and requested > 0: +- default_limit = requested ++ default_limit = min(default_limit, requested) ++ if verbose: ++ log( ++ "zip", ++ f"RAM detection failed and Max7zInst not set, using default worker limit → " ++ f"work items: {num_work_items}, requested: {requested}, final: {default_limit}", ++ verbose_only=True, ++ verbose=verbose ++ ) + return (default_limit, fixed_dict_size_bytes) + +``` + +
+ + + +--- + + + +--- + + +
+ Tool use: **read_lints** • Read lints for 1 file + +Lint paths: + +- `zip_sequences.py` + +Lint results + + +**No lint errors found** +
+ + +--- + + + + + +--- + + + +--- + + +Updated the code to cap worker count at the number of sequences to zip. Changes: + +1. **Auto-calculate mode (`Max7zInst=0`)**: Caps workers at the number of work items +2. **Configured mode (`Max7zInst=N`)**: Also caps at the number of work items +3. **Default fallback**: Also caps at the number of work items + +Now when there's only 1 sequence to zip, it will use 1 worker instead of trying to spawn 32. The verbose output will show: +- `work items: 1` +- `final: 1` (instead of 32) + +The script still uses aggressive RAM allocation when there are many sequences, but won't exceed the actual number of jobs. + +--- + diff --git a/zip_sequences.py b/zip_sequences.py index 4ae7101..6516eb1 100644 --- a/zip_sequences.py +++ b/zip_sequences.py @@ -264,8 +264,11 @@ def max_workers( # Check if Max7zInst is configured if MAX_7Z_INSTANCES is not None: - # Use configured maximum instances, but still respect user's --jobs + # Use configured maximum instances, but still respect user's --jobs and work items final_limit = MAX_7Z_INSTANCES + num_work_items = len(work_items) if work_items else 0 + if num_work_items > 0: + final_limit = min(final_limit, num_work_items) if requested and requested > 0: final_limit = min(final_limit, requested) @@ -273,7 +276,7 @@ def max_workers( log( "zip", f"Using Max7zInst={MAX_7Z_INSTANCES} from config → " - f"requested: {requested}, final: {final_limit}", + f"work items: {num_work_items}, requested: {requested}, final: {final_limit}", verbose_only=True, verbose=verbose ) @@ -281,18 +284,24 @@ def max_workers( return (final_limit, fixed_dict_size_bytes) # Auto-calculate based on RAM if Max7zInst not configured + # "Balls-to-the-walls" mode: use maximum resources if available_ram is not None: - # 7z uses ~2-3x dictionary size in RAM, but with overhead use 8x for safety - # This accounts for 7z's internal buffers, OS overhead, and other processes - FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 8 * 1024 * 1024 # 8GB per job + # 7z uses ~2-3x dictionary size in RAM, use 3x for aggressive mode + # This is more realistic and allows more concurrent workers + FIXED_RAM_PER_JOB = FIXED_DICT_SIZE_MB * 3 * 1024 * 1024 # 3GB per job # available_ram is already 80% of total (20% reserved for system) - # Use only 40% of that for compression jobs (very conservative to prevent swapping) - compression_ram = int(available_ram * 0.4) + # Use 95% of available RAM for compression jobs (aggressive mode) + compression_ram = int(available_ram * 0.95) # Calculate worker limit based on fixed per-job RAM ram_limit = max(1, compression_ram // FIXED_RAM_PER_JOB) + # Cap at number of actual work items (can't have more workers than jobs) + num_work_items = len(work_items) if work_items else 0 + if num_work_items > 0: + ram_limit = min(ram_limit, num_work_items) + # Use RAM limit directly (no CPU limit) final_limit = ram_limit if requested and requested > 0: @@ -304,8 +313,8 @@ def max_workers( ram_per_job_gb = FIXED_RAM_PER_JOB / (1024 ** 3) log( "zip", - f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (40%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " - f"RAM limit: {ram_limit}, requested: {requested}, final: {final_limit}", + f"RAM: {ram_gb:.1f}GB available (80% of total), {compression_ram_gb:.1f}GB for compression (95%), {ram_per_job_gb:.1f}GB per job (dict: {FIXED_DICT_SIZE_MB}MB) → " + f"RAM limit: {ram_limit}, work items: {num_work_items}, requested: {requested}, final: {final_limit}", verbose_only=True, verbose=verbose ) @@ -313,11 +322,20 @@ def max_workers( return (final_limit, fixed_dict_size_bytes) # RAM detection failed, use a safe default (no CPU limit) - if verbose: - log("zip", "RAM detection failed and Max7zInst not set, using default worker limit of 4", verbose_only=True, verbose=verbose) default_limit = 4 + num_work_items = len(work_items) if work_items else 0 + if num_work_items > 0: + default_limit = min(default_limit, num_work_items) if requested and requested > 0: - default_limit = requested + default_limit = min(default_limit, requested) + if verbose: + log( + "zip", + f"RAM detection failed and Max7zInst not set, using default worker limit → " + f"work items: {num_work_items}, requested: {requested}, final: {default_limit}", + verbose_only=True, + verbose=verbose + ) return (default_limit, fixed_dict_size_bytes) # For zip compression, use existing estimation-based approach