# Script to organize texture files by checksum with two-level duplicate detection # Pass 1: Intra-blendfile duplicates → [blendfile]\common # Pass 2: Inter-blendfile duplicates → \textures\common # Usage: .\organize_textures.ps1 # Prompt user for texture folder path $textureFolderPath = Read-Host "Enter texture folder path" # Validate the input path if ([string]::IsNullOrWhiteSpace($textureFolderPath)) { Write-Host "Error: No path provided." -ForegroundColor Red exit } if (-not (Test-Path -Path $textureFolderPath -PathType Container)) { Write-Host "Error: Path does not exist or is not a directory: $textureFolderPath" -ForegroundColor Red exit } # Resolve the full path $textureFolderPath = (Resolve-Path $textureFolderPath).ProviderPath Write-Host "Processing texture folder: $textureFolderPath" -ForegroundColor Cyan # Function to calculate checksums for files function Get-FilesWithChecksums { param( [array]$Files ) $throttleLimit = [Math]::Max(1, [Environment]::ProcessorCount) $parallelScriptBlock = { try { $hash = Get-FileHash -Path $_.FullName -Algorithm SHA256 [PSCustomObject]@{ File = $_ Hash = $hash.Hash } } catch { Write-Warning "Failed to calculate checksum for: $($_.FullName) - $($_.Exception.Message)" $null } } return $Files | ForEach-Object -Parallel $parallelScriptBlock -ThrottleLimit $throttleLimit | Where-Object { $null -ne $_ } } # Function to move files to common folder function Move-FilesToCommon { param( [array]$Files, [string]$CommonPath, [string]$DuplicatesPath, [hashtable]$FilesInCommon ) $movedCount = 0 $duplicateCount = 0 foreach ($fileObj in $Files) { $fileName = $fileObj.Name $destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName # Handle name conflicts if ($FilesInCommon.ContainsKey($fileName)) { $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName) $extension = [System.IO.Path]::GetExtension($fileName) $counter = 1 do { $newFileName = "${baseName}_${counter}${extension}" $destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName $counter++ } while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath)) $fileName = $newFileName } try { Move-Item -Path $fileObj.FullName -Destination $destinationPath -Force $FilesInCommon[$fileName] = $true $movedCount++ } catch { Write-Warning "Failed to move file: $($fileObj.FullName) - $($_.Exception.Message)" } } return @{ MovedCount = $movedCount DuplicateCount = $duplicateCount } } # Function to process duplicate group function Process-DuplicateGroup { param( [array]$Files, [string]$CommonPath, [string]$DuplicatesPath, [hashtable]$FilesInCommon ) $movedCount = 0 $duplicateCount = 0 if ($Files.Count -eq 1) { # Single file - leave in place (will be processed in Pass 2) return @{ MovedCount = 0 DuplicateCount = 0 } } # Multiple files with same checksum (duplicates) # Move first file to \common $firstFile = $Files[0].File $fileName = $firstFile.Name $destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName # Handle name conflicts for the first file if ($FilesInCommon.ContainsKey($fileName)) { $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName) $extension = [System.IO.Path]::GetExtension($fileName) $counter = 1 do { $newFileName = "${baseName}_${counter}${extension}" $destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName $counter++ } while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath)) $fileName = $newFileName } try { Move-Item -Path $firstFile.FullName -Destination $destinationPath -Force $FilesInCommon[$fileName] = $true $movedCount++ } catch { Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)" } # Move remaining files to \common\duplicates with numbered suffixes for ($i = 1; $i -lt $Files.Count; $i++) { $fileObj = $Files[$i].File $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name) $extension = [System.IO.Path]::GetExtension($fileObj.Name) $numberedFileName = "${baseName}.$($i.ToString('000'))${extension}" $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName # Handle name conflicts in duplicates folder $conflictCounter = 1 while (Test-Path -Path $duplicateDestinationPath) { $numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}" $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName $conflictCounter++ } try { Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force $duplicateCount++ } catch { Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)" } } return @{ MovedCount = $movedCount DuplicateCount = $duplicateCount } } # ============================================================================ # PASS 1: Intra-Blendfile Processing # ============================================================================ Write-Host "" Write-Host "=== PASS 1: Intra-Blendfile Processing ===" -ForegroundColor Cyan # Get all direct subdirectories of texture folder (blendfile folders) $blendfileFolders = Get-ChildItem -Path $textureFolderPath -Directory | Where-Object { $_.Name -ne "common" } if ($null -eq $blendfileFolders -or $blendfileFolders.Count -eq 0) { Write-Host "No blendfile folders found. Skipping Pass 1." -ForegroundColor Yellow } else { Write-Host "Found $($blendfileFolders.Count) blendfile folder(s) to process." -ForegroundColor Green $totalPass1Moved = 0 $totalPass1Duplicates = 0 foreach ($blendfileFolder in $blendfileFolders) { Write-Host "" Write-Host "Processing blendfile: $($blendfileFolder.Name)" -ForegroundColor Yellow # Get all files in this blendfile folder, excluding \common folders $blendfileFiles = Get-ChildItem -Path $blendfileFolder.FullName -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" } if ($null -eq $blendfileFiles -or $blendfileFiles.Count -eq 0) { Write-Host " No files found in this blendfile folder." -ForegroundColor Gray continue } Write-Host " Found $($blendfileFiles.Count) files." -ForegroundColor Gray # Calculate checksums Write-Host " Calculating checksums..." -ForegroundColor Gray $filesWithChecksums = Get-FilesWithChecksums -Files $blendfileFiles # Group by checksum $groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash Write-Host " Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Gray # Create [blendfile]\common and [blendfile]\common\duplicates directories $blendfileCommonPath = Join-Path -Path $blendfileFolder.FullName -ChildPath "common" $blendfileDuplicatesPath = Join-Path -Path $blendfileCommonPath -ChildPath "duplicates" if (-not (Test-Path -Path $blendfileCommonPath -PathType Container)) { New-Item -ItemType Directory -Path $blendfileCommonPath | Out-Null } if (-not (Test-Path -Path $blendfileDuplicatesPath -PathType Container)) { New-Item -ItemType Directory -Path $blendfileDuplicatesPath | Out-Null } # Track filenames already in [blendfile]\common $filesInBlendfileCommon = @{} # Process each checksum group $blendfileMoved = 0 $blendfileDuplicates = 0 foreach ($group in $groupedByChecksum) { $result = Process-DuplicateGroup -Files $group.Group -CommonPath $blendfileCommonPath -DuplicatesPath $blendfileDuplicatesPath -FilesInCommon $filesInBlendfileCommon $blendfileMoved += $result.MovedCount $blendfileDuplicates += $result.DuplicateCount } Write-Host " Moved $blendfileMoved file(s) to \common, $blendfileDuplicates duplicate(s) to \common\duplicates" -ForegroundColor Green $totalPass1Moved += $blendfileMoved $totalPass1Duplicates += $blendfileDuplicates } Write-Host "" Write-Host "Pass 1 complete: $totalPass1Moved file(s) moved, $totalPass1Duplicates duplicate(s) moved" -ForegroundColor Green } # ============================================================================ # PASS 2: Inter-Blendfile Processing # ============================================================================ Write-Host "" Write-Host "=== PASS 2: Inter-Blendfile Processing ===" -ForegroundColor Cyan # Get all remaining files (excluding all \common folders) Write-Host "Collecting remaining files..." -ForegroundColor Yellow $remainingFiles = Get-ChildItem -Path $textureFolderPath -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" } if ($null -eq $remainingFiles -or $remainingFiles.Count -eq 0) { Write-Host "No remaining files found to process." -ForegroundColor Yellow } else { Write-Host "Found $($remainingFiles.Count) remaining files to process." -ForegroundColor Green # Calculate checksums Write-Host "Calculating checksums using parallel processing (this may take a while)..." -ForegroundColor Yellow $filesWithChecksums = Get-FilesWithChecksums -Files $remainingFiles Write-Host "Checksum calculation complete." -ForegroundColor Green # Group files by checksum Write-Host "Grouping files by checksum..." -ForegroundColor Yellow $groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green # Create \textures\common and \textures\common\duplicates directories $rootCommonPath = Join-Path -Path $textureFolderPath -ChildPath "common" $rootDuplicatesPath = Join-Path -Path $rootCommonPath -ChildPath "duplicates" if (-not (Test-Path -Path $rootCommonPath -PathType Container)) { New-Item -ItemType Directory -Path $rootCommonPath | Out-Null Write-Host "Created directory: $rootCommonPath" -ForegroundColor Green } if (-not (Test-Path -Path $rootDuplicatesPath -PathType Container)) { New-Item -ItemType Directory -Path $rootDuplicatesPath | Out-Null Write-Host "Created directory: $rootDuplicatesPath" -ForegroundColor Green } # Track filenames already in \textures\common $filesInRootCommon = @{} # Process each checksum group Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow $pass2Moved = 0 $pass2Duplicates = 0 foreach ($group in $groupedByChecksum) { $files = $group.Group if ($files.Count -eq 1) { # Single file - leave in place (unique file) continue } # Multiple files with same checksum (duplicates across blendfiles) $result = Process-DuplicateGroup -Files $files -CommonPath $rootCommonPath -DuplicatesPath $rootDuplicatesPath -FilesInCommon $filesInRootCommon $pass2Moved += $result.MovedCount $pass2Duplicates += $result.DuplicateCount } Write-Host "" Write-Host "Pass 2 complete: $pass2Moved file(s) moved to \common, $pass2Duplicates duplicate(s) moved to \common\duplicates" -ForegroundColor Green } Write-Host "" Write-Host "File organization complete!" -ForegroundColor Green