separate behavior for inter- and intra-blendfile commonalities

.specstory/history/2025-12-19_17-59Z-texture-folder-commonality-script.md

@@ -3441,3 +3441,12 @@ Got it, no more problems detected.
 
 ---
 
+next issue: let's make sure it scans each folder separately. it goes \\textures\[Blendfile]\[Material]
+
+Intra-blendfile commonalities should go to `\\textures\[blendfile]\common`
+Inter-blendfile commonalities should go to `\\textures\common`
+
+sound good?
+
+---
+

organize_textures.ps1

@@ -1,4 +1,6 @@
-# Script to organize texture files by checksum, moving all files to \common and duplicates to \common\duplicates
+# Script to organize texture files by checksum with two-level duplicate detection
+# Pass 1: Intra-blendfile duplicates → [blendfile]\common
+# Pass 2: Inter-blendfile duplicates → \textures\common
 # Usage: .\organize_textures.ps1
 
 # Prompt user for texture folder path
@@ -19,153 +21,292 @@ if (-not (Test-Path -Path $textureFolderPath -PathType Container)) {
 $textureFolderPath = (Resolve-Path $textureFolderPath).ProviderPath
 Write-Host "Processing texture folder: $textureFolderPath" -ForegroundColor Cyan
 
-# Get all files recursively, excluding the \common folder to avoid processing already-moved files
+# Function to calculate checksums for files
-Write-Host "Collecting files..." -ForegroundColor Yellow
+function Get-FilesWithChecksums {
-$allFiles = Get-ChildItem -Path $textureFolderPath -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
+    param(
-
+        [array]$Files
-if ($null -eq $allFiles -or $allFiles.Count -eq 0) {
+    )
-    Write-Host "No files found to process (excluding \common folder)." -ForegroundColor Yellow
-    exit
-}
-
-Write-Host "Found $($allFiles.Count) files to process." -ForegroundColor Green
-
-# Calculate checksums for all files using parallel processing
-Write-Host "Calculating checksums using parallel processing (this may take a while)..." -ForegroundColor Yellow
-$throttleLimit = [Math]::Max(1, [Environment]::ProcessorCount)
-$parallelScriptBlock = {
-    try {
-        $hash = Get-FileHash -Path $_.FullName -Algorithm SHA256
-        [PSCustomObject]@{
-            File = $_
-            Hash = $hash.Hash
-        }
-    } catch {
-        Write-Warning "Failed to calculate checksum for: $($_.FullName) - $($_.Exception.Message)"
-        $null
-    }
-}
-$filesWithChecksums = $allFiles | ForEach-Object -Parallel $parallelScriptBlock -ThrottleLimit $throttleLimit | Where-Object { $null -ne $_ }
-
-Write-Host "Checksum calculation complete." -ForegroundColor Green
-
-# Group files by checksum
-Write-Host "Grouping files by checksum..." -ForegroundColor Yellow
-$groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
-
-Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green
-
-# Create \common and \common\duplicates directories
-$commonPath = Join-Path -Path $textureFolderPath -ChildPath "common"
-$duplicatesPath = Join-Path -Path $commonPath -ChildPath "duplicates"
-
-if (-not (Test-Path -Path $commonPath -PathType Container)) {
-    New-Item -ItemType Directory -Path $commonPath | Out-Null
-    Write-Host "Created directory: $commonPath" -ForegroundColor Green
-}
-
-if (-not (Test-Path -Path $duplicatesPath -PathType Container)) {
-    New-Item -ItemType Directory -Path $duplicatesPath | Out-Null
-    Write-Host "Created directory: $duplicatesPath" -ForegroundColor Green
-}
-
-# Track filenames already in \common to handle name conflicts
-$filesInCommon = @{}
-
-# Process each checksum group
-Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow
-$movedCount = 0
-$duplicateCount = 0
-
-foreach ($group in $groupedByChecksum) {
-    $files = $group.Group
     
-    if ($files.Count -eq 1) {
+    $throttleLimit = [Math]::Max(1, [Environment]::ProcessorCount)
-        # Single file - move to \common
+    $parallelScriptBlock = {
-        $fileObj = $files[0].File
+        try {
+            $hash = Get-FileHash -Path $_.FullName -Algorithm SHA256
+            [PSCustomObject]@{
+                File = $_
+                Hash = $hash.Hash
+            }
+        } catch {
+            Write-Warning "Failed to calculate checksum for: $($_.FullName) - $($_.Exception.Message)"
+            $null
+        }
+    }
+    
+    return $Files | ForEach-Object -Parallel $parallelScriptBlock -ThrottleLimit $throttleLimit | Where-Object { $null -ne $_ }
+}
+
+# Function to move files to common folder
+function Move-FilesToCommon {
+    param(
+        [array]$Files,
+        [string]$CommonPath,
+        [string]$DuplicatesPath,
+        [hashtable]$FilesInCommon
+    )
+    
+    $movedCount = 0
+    $duplicateCount = 0
+    
+    foreach ($fileObj in $Files) {
         $fileName = $fileObj.Name
-        $destinationPath = Join-Path -Path $commonPath -ChildPath $fileName
+        $destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName
         
         # Handle name conflicts
-        if ($filesInCommon.ContainsKey($fileName)) {
+        if ($FilesInCommon.ContainsKey($fileName)) {
-            # File with same name but different checksum already exists
             $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
             $extension = [System.IO.Path]::GetExtension($fileName)
             $counter = 1
             do {
                 $newFileName = "${baseName}_${counter}${extension}"
-                $destinationPath = Join-Path -Path $commonPath -ChildPath $newFileName
+                $destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName
                 $counter++
-            } while ($filesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
+            } while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
             
             $fileName = $newFileName
-            Write-Host "  Name conflict resolved: renamed to '$fileName'" -ForegroundColor Yellow
         }
         
         try {
             Move-Item -Path $fileObj.FullName -Destination $destinationPath -Force
-            $filesInCommon[$fileName] = $true
+            $FilesInCommon[$fileName] = $true
             $movedCount++
         } catch {
             Write-Warning "Failed to move file: $($fileObj.FullName) - $($_.Exception.Message)"
         }
-    } else {
+    }
-        # Multiple files with same checksum (duplicates)
+    
-        # Move first file to \common
+    return @{
-        $firstFile = $files[0].File
+        MovedCount = $movedCount
-        $fileName = $firstFile.Name
+        DuplicateCount = $duplicateCount
-        $destinationPath = Join-Path -Path $commonPath -ChildPath $fileName
+    }
+}
+
+# Function to process duplicate group
+function Process-DuplicateGroup {
+    param(
+        [array]$Files,
+        [string]$CommonPath,
+        [string]$DuplicatesPath,
+        [hashtable]$FilesInCommon
+    )
+    
+    $movedCount = 0
+    $duplicateCount = 0
+    
+    if ($Files.Count -eq 1) {
+        # Single file - leave in place (will be processed in Pass 2)
+        return @{
+            MovedCount = 0
+            DuplicateCount = 0
+        }
+    }
+    
+    # Multiple files with same checksum (duplicates)
+    # Move first file to \common
+    $firstFile = $Files[0].File
+    $fileName = $firstFile.Name
+    $destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName
+    
+    # Handle name conflicts for the first file
+    if ($FilesInCommon.ContainsKey($fileName)) {
+        $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
+        $extension = [System.IO.Path]::GetExtension($fileName)
+        $counter = 1
+        do {
+            $newFileName = "${baseName}_${counter}${extension}"
+            $destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName
+            $counter++
+        } while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
         
-        # Handle name conflicts for the first file
+        $fileName = $newFileName
-        if ($filesInCommon.ContainsKey($fileName)) {
+    }
-            $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
+    
-            $extension = [System.IO.Path]::GetExtension($fileName)
+    try {
-            $counter = 1
+        Move-Item -Path $firstFile.FullName -Destination $destinationPath -Force
-            do {
+        $FilesInCommon[$fileName] = $true
-                $newFileName = "${baseName}_${counter}${extension}"
+        $movedCount++
-                $destinationPath = Join-Path -Path $commonPath -ChildPath $newFileName
+    } catch {
-                $counter++
+        Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)"
-            } while ($filesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
+    }
-            
+    
-            $fileName = $newFileName
+    # Move remaining files to \common\duplicates with numbered suffixes
-            Write-Host "  Name conflict resolved for duplicate group: renamed to '$fileName'" -ForegroundColor Yellow
+    for ($i = 1; $i -lt $Files.Count; $i++) {
+        $fileObj = $Files[$i].File
+        $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name)
+        $extension = [System.IO.Path]::GetExtension($fileObj.Name)
+        $numberedFileName = "${baseName}.$($i.ToString('000'))${extension}"
+        $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
+        
+        # Handle name conflicts in duplicates folder
+        $conflictCounter = 1
+        while (Test-Path -Path $duplicateDestinationPath) {
+            $numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}"
+            $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
+            $conflictCounter++
         }
         
         try {
-            Move-Item -Path $firstFile.FullName -Destination $destinationPath -Force
+            Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force
-            $filesInCommon[$fileName] = $true
+            $duplicateCount++
-            $movedCount++
         } catch {
-            Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)"
+            Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
-        }
-        
-        # Move remaining files to \common\duplicates with numbered suffixes
-        for ($i = 1; $i -lt $files.Count; $i++) {
-            $fileObj = $files[$i].File
-            $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name)
-            $extension = [System.IO.Path]::GetExtension($fileObj.Name)
-            $numberedFileName = "${baseName}.$($i.ToString('000'))${extension}"
-            $duplicateDestinationPath = Join-Path -Path $duplicatesPath -ChildPath $numberedFileName
-            
-            # Handle name conflicts in duplicates folder
-            $conflictCounter = 1
-            while (Test-Path -Path $duplicateDestinationPath) {
-                $numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}"
-                $duplicateDestinationPath = Join-Path -Path $duplicatesPath -ChildPath $numberedFileName
-                $conflictCounter++
-            }
-            
-            try {
-                Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force
-                $duplicateCount++
-            } catch {
-                Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
-            }
         }
     }
+    
+    return @{
+        MovedCount = $movedCount
+        DuplicateCount = $duplicateCount
+    }
+}
+
+# ============================================================================
+# PASS 1: Intra-Blendfile Processing
+# ============================================================================
+Write-Host ""
+Write-Host "=== PASS 1: Intra-Blendfile Processing ===" -ForegroundColor Cyan
+
+# Get all direct subdirectories of texture folder (blendfile folders)
+$blendfileFolders = Get-ChildItem -Path $textureFolderPath -Directory | Where-Object { $_.Name -ne "common" }
+
+if ($null -eq $blendfileFolders -or $blendfileFolders.Count -eq 0) {
+    Write-Host "No blendfile folders found. Skipping Pass 1." -ForegroundColor Yellow
+} else {
+    Write-Host "Found $($blendfileFolders.Count) blendfile folder(s) to process." -ForegroundColor Green
+    
+    $totalPass1Moved = 0
+    $totalPass1Duplicates = 0
+    
+    foreach ($blendfileFolder in $blendfileFolders) {
+        Write-Host ""
+        Write-Host "Processing blendfile: $($blendfileFolder.Name)" -ForegroundColor Yellow
+        
+        # Get all files in this blendfile folder, excluding \common folders
+        $blendfileFiles = Get-ChildItem -Path $blendfileFolder.FullName -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
+        
+        if ($null -eq $blendfileFiles -or $blendfileFiles.Count -eq 0) {
+            Write-Host "  No files found in this blendfile folder." -ForegroundColor Gray
+            continue
+        }
+        
+        Write-Host "  Found $($blendfileFiles.Count) files." -ForegroundColor Gray
+        
+        # Calculate checksums
+        Write-Host "  Calculating checksums..." -ForegroundColor Gray
+        $filesWithChecksums = Get-FilesWithChecksums -Files $blendfileFiles
+        
+        # Group by checksum
+        $groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
+        
+        Write-Host "  Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Gray
+        
+        # Create [blendfile]\common and [blendfile]\common\duplicates directories
+        $blendfileCommonPath = Join-Path -Path $blendfileFolder.FullName -ChildPath "common"
+        $blendfileDuplicatesPath = Join-Path -Path $blendfileCommonPath -ChildPath "duplicates"
+        
+        if (-not (Test-Path -Path $blendfileCommonPath -PathType Container)) {
+            New-Item -ItemType Directory -Path $blendfileCommonPath | Out-Null
+        }
+        
+        if (-not (Test-Path -Path $blendfileDuplicatesPath -PathType Container)) {
+            New-Item -ItemType Directory -Path $blendfileDuplicatesPath | Out-Null
+        }
+        
+        # Track filenames already in [blendfile]\common
+        $filesInBlendfileCommon = @{}
+        
+        # Process each checksum group
+        $blendfileMoved = 0
+        $blendfileDuplicates = 0
+        
+        foreach ($group in $groupedByChecksum) {
+            $result = Process-DuplicateGroup -Files $group.Group -CommonPath $blendfileCommonPath -DuplicatesPath $blendfileDuplicatesPath -FilesInCommon $filesInBlendfileCommon
+            $blendfileMoved += $result.MovedCount
+            $blendfileDuplicates += $result.DuplicateCount
+        }
+        
+        Write-Host "  Moved $blendfileMoved file(s) to \common, $blendfileDuplicates duplicate(s) to \common\duplicates" -ForegroundColor Green
+        $totalPass1Moved += $blendfileMoved
+        $totalPass1Duplicates += $blendfileDuplicates
+    }
+    
+    Write-Host ""
+    Write-Host "Pass 1 complete: $totalPass1Moved file(s) moved, $totalPass1Duplicates duplicate(s) moved" -ForegroundColor Green
+}
+
+# ============================================================================
+# PASS 2: Inter-Blendfile Processing
+# ============================================================================
+Write-Host ""
+Write-Host "=== PASS 2: Inter-Blendfile Processing ===" -ForegroundColor Cyan
+
+# Get all remaining files (excluding all \common folders)
+Write-Host "Collecting remaining files..." -ForegroundColor Yellow
+$remainingFiles = Get-ChildItem -Path $textureFolderPath -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
+
+if ($null -eq $remainingFiles -or $remainingFiles.Count -eq 0) {
+    Write-Host "No remaining files found to process." -ForegroundColor Yellow
+} else {
+    Write-Host "Found $($remainingFiles.Count) remaining files to process." -ForegroundColor Green
+    
+    # Calculate checksums
+    Write-Host "Calculating checksums using parallel processing (this may take a while)..." -ForegroundColor Yellow
+    $filesWithChecksums = Get-FilesWithChecksums -Files $remainingFiles
+    
+    Write-Host "Checksum calculation complete." -ForegroundColor Green
+    
+    # Group files by checksum
+    Write-Host "Grouping files by checksum..." -ForegroundColor Yellow
+    $groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
+    
+    Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green
+    
+    # Create \textures\common and \textures\common\duplicates directories
+    $rootCommonPath = Join-Path -Path $textureFolderPath -ChildPath "common"
+    $rootDuplicatesPath = Join-Path -Path $rootCommonPath -ChildPath "duplicates"
+    
+    if (-not (Test-Path -Path $rootCommonPath -PathType Container)) {
+        New-Item -ItemType Directory -Path $rootCommonPath | Out-Null
+        Write-Host "Created directory: $rootCommonPath" -ForegroundColor Green
+    }
+    
+    if (-not (Test-Path -Path $rootDuplicatesPath -PathType Container)) {
+        New-Item -ItemType Directory -Path $rootDuplicatesPath | Out-Null
+        Write-Host "Created directory: $rootDuplicatesPath" -ForegroundColor Green
+    }
+    
+    # Track filenames already in \textures\common
+    $filesInRootCommon = @{}
+    
+    # Process each checksum group
+    Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow
+    $pass2Moved = 0
+    $pass2Duplicates = 0
+    
+    foreach ($group in $groupedByChecksum) {
+        $files = $group.Group
+        
+        if ($files.Count -eq 1) {
+            # Single file - leave in place (unique file)
+            continue
+        }
+        
+        # Multiple files with same checksum (duplicates across blendfiles)
+        $result = Process-DuplicateGroup -Files $files -CommonPath $rootCommonPath -DuplicatesPath $rootDuplicatesPath -FilesInCommon $filesInRootCommon
+        $pass2Moved += $result.MovedCount
+        $pass2Duplicates += $result.DuplicateCount
+    }
+    
+    Write-Host ""
+    Write-Host "Pass 2 complete: $pass2Moved file(s) moved to \common, $pass2Duplicates duplicate(s) moved to \common\duplicates" -ForegroundColor Green
 }
 
 Write-Host ""
 Write-Host "File organization complete!" -ForegroundColor Green
-Write-Host "  Files moved to \common: $movedCount" -ForegroundColor Cyan
-Write-Host "  Duplicate files moved to \common\duplicates: $duplicateCount" -ForegroundColor Cyan