separate behavior for inter- and intra-blendfile commonalities
This commit is contained in:
Nathan
@@ -3441,3 +3441,12 @@ Got it, no more problems detected.
|
||||
|
||||
---
|
||||
|
||||
next issue: let's make sure it scans each folder separately. it goes \\textures\[Blendfile]\[Material]
|
||||
|
||||
Intra-blendfile commonalities should go to `\\textures\[blendfile]\common`
|
||||
Inter-blendfile commonalities should go to `\\textures\common`
|
||||
|
||||
sound good?
|
||||
|
||||
---
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,6 @@
|
||||
# Script to organize texture files by checksum, moving all files to \common and duplicates to \common\duplicates
|
||||
# Script to organize texture files by checksum with two-level duplicate detection
|
||||
# Pass 1: Intra-blendfile duplicates → [blendfile]\common
|
||||
# Pass 2: Inter-blendfile duplicates → \textures\common
|
||||
# Usage: .\organize_textures.ps1
|
||||
|
||||
# Prompt user for texture folder path
|
||||
@@ -19,153 +21,292 @@ if (-not (Test-Path -Path $textureFolderPath -PathType Container)) {
|
||||
$textureFolderPath = (Resolve-Path $textureFolderPath).ProviderPath
|
||||
Write-Host "Processing texture folder: $textureFolderPath" -ForegroundColor Cyan
|
||||
|
||||
# Get all files recursively, excluding the \common folder to avoid processing already-moved files
|
||||
Write-Host "Collecting files..." -ForegroundColor Yellow
|
||||
$allFiles = Get-ChildItem -Path $textureFolderPath -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
|
||||
# Function to calculate checksums for files
|
||||
function Get-FilesWithChecksums {
|
||||
param(
|
||||
[array]$Files
|
||||
)
|
||||
|
||||
if ($null -eq $allFiles -or $allFiles.Count -eq 0) {
|
||||
Write-Host "No files found to process (excluding \common folder)." -ForegroundColor Yellow
|
||||
exit
|
||||
}
|
||||
|
||||
Write-Host "Found $($allFiles.Count) files to process." -ForegroundColor Green
|
||||
|
||||
# Calculate checksums for all files using parallel processing
|
||||
Write-Host "Calculating checksums using parallel processing (this may take a while)..." -ForegroundColor Yellow
|
||||
$throttleLimit = [Math]::Max(1, [Environment]::ProcessorCount)
|
||||
$parallelScriptBlock = {
|
||||
try {
|
||||
$hash = Get-FileHash -Path $_.FullName -Algorithm SHA256
|
||||
[PSCustomObject]@{
|
||||
File = $_
|
||||
Hash = $hash.Hash
|
||||
$throttleLimit = [Math]::Max(1, [Environment]::ProcessorCount)
|
||||
$parallelScriptBlock = {
|
||||
try {
|
||||
$hash = Get-FileHash -Path $_.FullName -Algorithm SHA256
|
||||
[PSCustomObject]@{
|
||||
File = $_
|
||||
Hash = $hash.Hash
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "Failed to calculate checksum for: $($_.FullName) - $($_.Exception.Message)"
|
||||
$null
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "Failed to calculate checksum for: $($_.FullName) - $($_.Exception.Message)"
|
||||
$null
|
||||
}
|
||||
}
|
||||
$filesWithChecksums = $allFiles | ForEach-Object -Parallel $parallelScriptBlock -ThrottleLimit $throttleLimit | Where-Object { $null -ne $_ }
|
||||
|
||||
Write-Host "Checksum calculation complete." -ForegroundColor Green
|
||||
|
||||
# Group files by checksum
|
||||
Write-Host "Grouping files by checksum..." -ForegroundColor Yellow
|
||||
$groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
|
||||
|
||||
Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green
|
||||
|
||||
# Create \common and \common\duplicates directories
|
||||
$commonPath = Join-Path -Path $textureFolderPath -ChildPath "common"
|
||||
$duplicatesPath = Join-Path -Path $commonPath -ChildPath "duplicates"
|
||||
|
||||
if (-not (Test-Path -Path $commonPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $commonPath | Out-Null
|
||||
Write-Host "Created directory: $commonPath" -ForegroundColor Green
|
||||
return $Files | ForEach-Object -Parallel $parallelScriptBlock -ThrottleLimit $throttleLimit | Where-Object { $null -ne $_ }
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $duplicatesPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $duplicatesPath | Out-Null
|
||||
Write-Host "Created directory: $duplicatesPath" -ForegroundColor Green
|
||||
}
|
||||
# Function to move files to common folder
|
||||
function Move-FilesToCommon {
|
||||
param(
|
||||
[array]$Files,
|
||||
[string]$CommonPath,
|
||||
[string]$DuplicatesPath,
|
||||
[hashtable]$FilesInCommon
|
||||
)
|
||||
|
||||
# Track filenames already in \common to handle name conflicts
|
||||
$filesInCommon = @{}
|
||||
$movedCount = 0
|
||||
$duplicateCount = 0
|
||||
|
||||
# Process each checksum group
|
||||
Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow
|
||||
$movedCount = 0
|
||||
$duplicateCount = 0
|
||||
|
||||
foreach ($group in $groupedByChecksum) {
|
||||
$files = $group.Group
|
||||
|
||||
if ($files.Count -eq 1) {
|
||||
# Single file - move to \common
|
||||
$fileObj = $files[0].File
|
||||
foreach ($fileObj in $Files) {
|
||||
$fileName = $fileObj.Name
|
||||
$destinationPath = Join-Path -Path $commonPath -ChildPath $fileName
|
||||
$destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName
|
||||
|
||||
# Handle name conflicts
|
||||
if ($filesInCommon.ContainsKey($fileName)) {
|
||||
# File with same name but different checksum already exists
|
||||
if ($FilesInCommon.ContainsKey($fileName)) {
|
||||
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
|
||||
$extension = [System.IO.Path]::GetExtension($fileName)
|
||||
$counter = 1
|
||||
do {
|
||||
$newFileName = "${baseName}_${counter}${extension}"
|
||||
$destinationPath = Join-Path -Path $commonPath -ChildPath $newFileName
|
||||
$destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName
|
||||
$counter++
|
||||
} while ($filesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
|
||||
} while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
|
||||
|
||||
$fileName = $newFileName
|
||||
Write-Host " Name conflict resolved: renamed to '$fileName'" -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
try {
|
||||
Move-Item -Path $fileObj.FullName -Destination $destinationPath -Force
|
||||
$filesInCommon[$fileName] = $true
|
||||
$FilesInCommon[$fileName] = $true
|
||||
$movedCount++
|
||||
} catch {
|
||||
Write-Warning "Failed to move file: $($fileObj.FullName) - $($_.Exception.Message)"
|
||||
}
|
||||
} else {
|
||||
# Multiple files with same checksum (duplicates)
|
||||
# Move first file to \common
|
||||
$firstFile = $files[0].File
|
||||
$fileName = $firstFile.Name
|
||||
$destinationPath = Join-Path -Path $commonPath -ChildPath $fileName
|
||||
}
|
||||
|
||||
# Handle name conflicts for the first file
|
||||
if ($filesInCommon.ContainsKey($fileName)) {
|
||||
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
|
||||
$extension = [System.IO.Path]::GetExtension($fileName)
|
||||
$counter = 1
|
||||
do {
|
||||
$newFileName = "${baseName}_${counter}${extension}"
|
||||
$destinationPath = Join-Path -Path $commonPath -ChildPath $newFileName
|
||||
$counter++
|
||||
} while ($filesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
|
||||
return @{
|
||||
MovedCount = $movedCount
|
||||
DuplicateCount = $duplicateCount
|
||||
}
|
||||
}
|
||||
|
||||
$fileName = $newFileName
|
||||
Write-Host " Name conflict resolved for duplicate group: renamed to '$fileName'" -ForegroundColor Yellow
|
||||
# Function to process duplicate group
|
||||
function Process-DuplicateGroup {
|
||||
param(
|
||||
[array]$Files,
|
||||
[string]$CommonPath,
|
||||
[string]$DuplicatesPath,
|
||||
[hashtable]$FilesInCommon
|
||||
)
|
||||
|
||||
$movedCount = 0
|
||||
$duplicateCount = 0
|
||||
|
||||
if ($Files.Count -eq 1) {
|
||||
# Single file - leave in place (will be processed in Pass 2)
|
||||
return @{
|
||||
MovedCount = 0
|
||||
DuplicateCount = 0
|
||||
}
|
||||
}
|
||||
|
||||
# Multiple files with same checksum (duplicates)
|
||||
# Move first file to \common
|
||||
$firstFile = $Files[0].File
|
||||
$fileName = $firstFile.Name
|
||||
$destinationPath = Join-Path -Path $CommonPath -ChildPath $fileName
|
||||
|
||||
# Handle name conflicts for the first file
|
||||
if ($FilesInCommon.ContainsKey($fileName)) {
|
||||
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileName)
|
||||
$extension = [System.IO.Path]::GetExtension($fileName)
|
||||
$counter = 1
|
||||
do {
|
||||
$newFileName = "${baseName}_${counter}${extension}"
|
||||
$destinationPath = Join-Path -Path $CommonPath -ChildPath $newFileName
|
||||
$counter++
|
||||
} while ($FilesInCommon.ContainsKey($newFileName) -or (Test-Path -Path $destinationPath))
|
||||
|
||||
$fileName = $newFileName
|
||||
}
|
||||
|
||||
try {
|
||||
Move-Item -Path $firstFile.FullName -Destination $destinationPath -Force
|
||||
$FilesInCommon[$fileName] = $true
|
||||
$movedCount++
|
||||
} catch {
|
||||
Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
# Move remaining files to \common\duplicates with numbered suffixes
|
||||
for ($i = 1; $i -lt $Files.Count; $i++) {
|
||||
$fileObj = $Files[$i].File
|
||||
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name)
|
||||
$extension = [System.IO.Path]::GetExtension($fileObj.Name)
|
||||
$numberedFileName = "${baseName}.$($i.ToString('000'))${extension}"
|
||||
$duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
|
||||
|
||||
# Handle name conflicts in duplicates folder
|
||||
$conflictCounter = 1
|
||||
while (Test-Path -Path $duplicateDestinationPath) {
|
||||
$numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}"
|
||||
$duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
|
||||
$conflictCounter++
|
||||
}
|
||||
|
||||
try {
|
||||
Move-Item -Path $firstFile.FullName -Destination $destinationPath -Force
|
||||
$filesInCommon[$fileName] = $true
|
||||
$movedCount++
|
||||
Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force
|
||||
$duplicateCount++
|
||||
} catch {
|
||||
Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
# Move remaining files to \common\duplicates with numbered suffixes
|
||||
for ($i = 1; $i -lt $files.Count; $i++) {
|
||||
$fileObj = $files[$i].File
|
||||
$baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name)
|
||||
$extension = [System.IO.Path]::GetExtension($fileObj.Name)
|
||||
$numberedFileName = "${baseName}.$($i.ToString('000'))${extension}"
|
||||
$duplicateDestinationPath = Join-Path -Path $duplicatesPath -ChildPath $numberedFileName
|
||||
|
||||
# Handle name conflicts in duplicates folder
|
||||
$conflictCounter = 1
|
||||
while (Test-Path -Path $duplicateDestinationPath) {
|
||||
$numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}"
|
||||
$duplicateDestinationPath = Join-Path -Path $duplicatesPath -ChildPath $numberedFileName
|
||||
$conflictCounter++
|
||||
}
|
||||
|
||||
try {
|
||||
Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force
|
||||
$duplicateCount++
|
||||
} catch {
|
||||
Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
|
||||
}
|
||||
Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
return @{
|
||||
MovedCount = $movedCount
|
||||
DuplicateCount = $duplicateCount
|
||||
}
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PASS 1: Intra-Blendfile Processing
|
||||
# ============================================================================
|
||||
Write-Host ""
|
||||
Write-Host "=== PASS 1: Intra-Blendfile Processing ===" -ForegroundColor Cyan
|
||||
|
||||
# Get all direct subdirectories of texture folder (blendfile folders)
|
||||
$blendfileFolders = Get-ChildItem -Path $textureFolderPath -Directory | Where-Object { $_.Name -ne "common" }
|
||||
|
||||
if ($null -eq $blendfileFolders -or $blendfileFolders.Count -eq 0) {
|
||||
Write-Host "No blendfile folders found. Skipping Pass 1." -ForegroundColor Yellow
|
||||
} else {
|
||||
Write-Host "Found $($blendfileFolders.Count) blendfile folder(s) to process." -ForegroundColor Green
|
||||
|
||||
$totalPass1Moved = 0
|
||||
$totalPass1Duplicates = 0
|
||||
|
||||
foreach ($blendfileFolder in $blendfileFolders) {
|
||||
Write-Host ""
|
||||
Write-Host "Processing blendfile: $($blendfileFolder.Name)" -ForegroundColor Yellow
|
||||
|
||||
# Get all files in this blendfile folder, excluding \common folders
|
||||
$blendfileFiles = Get-ChildItem -Path $blendfileFolder.FullName -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
|
||||
|
||||
if ($null -eq $blendfileFiles -or $blendfileFiles.Count -eq 0) {
|
||||
Write-Host " No files found in this blendfile folder." -ForegroundColor Gray
|
||||
continue
|
||||
}
|
||||
|
||||
Write-Host " Found $($blendfileFiles.Count) files." -ForegroundColor Gray
|
||||
|
||||
# Calculate checksums
|
||||
Write-Host " Calculating checksums..." -ForegroundColor Gray
|
||||
$filesWithChecksums = Get-FilesWithChecksums -Files $blendfileFiles
|
||||
|
||||
# Group by checksum
|
||||
$groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
|
||||
|
||||
Write-Host " Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Gray
|
||||
|
||||
# Create [blendfile]\common and [blendfile]\common\duplicates directories
|
||||
$blendfileCommonPath = Join-Path -Path $blendfileFolder.FullName -ChildPath "common"
|
||||
$blendfileDuplicatesPath = Join-Path -Path $blendfileCommonPath -ChildPath "duplicates"
|
||||
|
||||
if (-not (Test-Path -Path $blendfileCommonPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $blendfileCommonPath | Out-Null
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $blendfileDuplicatesPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $blendfileDuplicatesPath | Out-Null
|
||||
}
|
||||
|
||||
# Track filenames already in [blendfile]\common
|
||||
$filesInBlendfileCommon = @{}
|
||||
|
||||
# Process each checksum group
|
||||
$blendfileMoved = 0
|
||||
$blendfileDuplicates = 0
|
||||
|
||||
foreach ($group in $groupedByChecksum) {
|
||||
$result = Process-DuplicateGroup -Files $group.Group -CommonPath $blendfileCommonPath -DuplicatesPath $blendfileDuplicatesPath -FilesInCommon $filesInBlendfileCommon
|
||||
$blendfileMoved += $result.MovedCount
|
||||
$blendfileDuplicates += $result.DuplicateCount
|
||||
}
|
||||
|
||||
Write-Host " Moved $blendfileMoved file(s) to \common, $blendfileDuplicates duplicate(s) to \common\duplicates" -ForegroundColor Green
|
||||
$totalPass1Moved += $blendfileMoved
|
||||
$totalPass1Duplicates += $blendfileDuplicates
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Pass 1 complete: $totalPass1Moved file(s) moved, $totalPass1Duplicates duplicate(s) moved" -ForegroundColor Green
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# PASS 2: Inter-Blendfile Processing
|
||||
# ============================================================================
|
||||
Write-Host ""
|
||||
Write-Host "=== PASS 2: Inter-Blendfile Processing ===" -ForegroundColor Cyan
|
||||
|
||||
# Get all remaining files (excluding all \common folders)
|
||||
Write-Host "Collecting remaining files..." -ForegroundColor Yellow
|
||||
$remainingFiles = Get-ChildItem -Path $textureFolderPath -Recurse -File | Where-Object { $_.FullName -notlike "*\common\*" }
|
||||
|
||||
if ($null -eq $remainingFiles -or $remainingFiles.Count -eq 0) {
|
||||
Write-Host "No remaining files found to process." -ForegroundColor Yellow
|
||||
} else {
|
||||
Write-Host "Found $($remainingFiles.Count) remaining files to process." -ForegroundColor Green
|
||||
|
||||
# Calculate checksums
|
||||
Write-Host "Calculating checksums using parallel processing (this may take a while)..." -ForegroundColor Yellow
|
||||
$filesWithChecksums = Get-FilesWithChecksums -Files $remainingFiles
|
||||
|
||||
Write-Host "Checksum calculation complete." -ForegroundColor Green
|
||||
|
||||
# Group files by checksum
|
||||
Write-Host "Grouping files by checksum..." -ForegroundColor Yellow
|
||||
$groupedByChecksum = $filesWithChecksums | Group-Object -Property Hash
|
||||
|
||||
Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green
|
||||
|
||||
# Create \textures\common and \textures\common\duplicates directories
|
||||
$rootCommonPath = Join-Path -Path $textureFolderPath -ChildPath "common"
|
||||
$rootDuplicatesPath = Join-Path -Path $rootCommonPath -ChildPath "duplicates"
|
||||
|
||||
if (-not (Test-Path -Path $rootCommonPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $rootCommonPath | Out-Null
|
||||
Write-Host "Created directory: $rootCommonPath" -ForegroundColor Green
|
||||
}
|
||||
|
||||
if (-not (Test-Path -Path $rootDuplicatesPath -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $rootDuplicatesPath | Out-Null
|
||||
Write-Host "Created directory: $rootDuplicatesPath" -ForegroundColor Green
|
||||
}
|
||||
|
||||
# Track filenames already in \textures\common
|
||||
$filesInRootCommon = @{}
|
||||
|
||||
# Process each checksum group
|
||||
Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow
|
||||
$pass2Moved = 0
|
||||
$pass2Duplicates = 0
|
||||
|
||||
foreach ($group in $groupedByChecksum) {
|
||||
$files = $group.Group
|
||||
|
||||
if ($files.Count -eq 1) {
|
||||
# Single file - leave in place (unique file)
|
||||
continue
|
||||
}
|
||||
|
||||
# Multiple files with same checksum (duplicates across blendfiles)
|
||||
$result = Process-DuplicateGroup -Files $files -CommonPath $rootCommonPath -DuplicatesPath $rootDuplicatesPath -FilesInCommon $filesInRootCommon
|
||||
$pass2Moved += $result.MovedCount
|
||||
$pass2Duplicates += $result.DuplicateCount
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Pass 2 complete: $pass2Moved file(s) moved to \common, $pass2Duplicates duplicate(s) moved to \common\duplicates" -ForegroundColor Green
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "File organization complete!" -ForegroundColor Green
|
||||
Write-Host " Files moved to \common: $movedCount" -ForegroundColor Cyan
|
||||
Write-Host " Duplicate files moved to \common\duplicates: $duplicateCount" -ForegroundColor Cyan
|
||||
|
||||
Reference in New Issue
Block a user