remove duplicate files

2025-12-19 13:20:58 -07:00
parent 5232e7c2cc
commit 5a7d1f66b9
2 changed files with 369 additions and 36 deletions
--- a/.specstory/history/2025-12-19_18-52Z-texture-commonality-folder-structure.md
+++ b/.specstory/history/2025-12-19_18-52Z-texture-commonality-folder-structure.md
@@ -7021,3 +7021,361 @@ _**User**_

 ---

+_**Agent (model default, mode Agent)**_
+
+
+The function is stripping the first underscore-separated part regardless of whether it's a blendfile prefix. It's removing descriptors like "Loafers_", "Black_", "BlueJeans_", "BluePolo_", "Scrub_", "Work_", "Cargo_" that should remain.
+
+We should only strip known blendfile prefixes (e.g., "Demarco_", "Chan_", "Joe_", "Dennis_", "Kirk_", "Martha_", "Hailey_", "Regina_", "Manny_"). Updating the function to check against a list of valid blendfile prefixes:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Checking how blendfile folders are identified to build a list of valid prefixes:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Updating the function to only strip known blendfile prefixes. Building the list from the blendfile folder names and passing it to the function:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Updating the function to only strip known blendfile prefixes. Modifying `Get-FileNameWithoutPrefix` to accept a list of valid prefixes and updating the calls:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "d56bed0f-1611-4026-b349-1367f22a00f5"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+Updating `Process-DuplicateGroup` to accept and use the valid prefixes list:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Updating `Process-DuplicateGroup` to accept and use the valid prefixes:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "7e9cab73-2e6a-4c51-aa15-d9ddc71f8fb2"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+Updating calls to `Get-FileNameWithoutPrefix` within `Process-DuplicateGroup`:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "50bf96a9-7c22-49e2-b1f6-6ee8fa110e7b"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+Building the list of valid prefixes before Pass 2 and passing it to `Process-DuplicateGroup`:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "67686096-d013-4c0a-bf48-a813ecc99452"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+Updating the Pass 2 call to `Process-DuplicateGroup` to pass the valid prefixes:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Updating the call to include the valid prefixes:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "2bd83a67-0dd8-46fc-9588-07244da0e8a3"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+The script now:
+
+1. Builds a list of valid blendfile prefixes from the folder names (Chan, Demarco, Dennis, Joe, Kirk, Martha, Hailey, Regina, Manny, etc.)
+2. Only strips prefixes that are in this list
+3. Leaves other prefixes like "Scrub_", "Work_", "Cargo_", "Black_", "BlueJeans_", "BluePolo_", "Loafers_" intact since they're part of the actual filename, not blendfile prefixes
+
+The changes:
+- Updated `Get-FileNameWithoutPrefix` to accept a `$ValidPrefixes` parameter and only strip prefixes in that list
+- Updated `Process-DuplicateGroup` to accept and pass `$ValidPrefixes`
+- Build the list of valid prefixes from blendfile folder names before Pass 2
+- Pass the valid prefixes when calling `Process-DuplicateGroup` with `-StripPrefix`
+
+When you run the script, it will only strip actual blendfile prefixes (e.g., "Demarco_", "Chan_") and preserve other prefixes that are part of the filename (e.g., "Loafers_", "Black_", "Scrub_").
+
+---
+
+_**User**_
+
+@pwsh (358-1075) that's a good result! next, we need to remove the duplicates subfolders, since that was only for debugging purposes. duplicates should be removed outright, because they're replaced by the common file (that's the point; saves space)
+
+---
+
+_**Agent (model default, mode Agent)**_
+
+
+Updating the script to delete duplicate files instead of moving them to a duplicates folder. Removing duplicates folder creation and updating the logic:
+
+
+---
+
+<tool-use data-tool-type="read" data-tool-name="read_file_v2">
+<details><summary>Tool use: **read_file_v2** • Read file: a:\1 Amazon_Active_Projects\3 ProjectStructure\organize_textures.ps1</summary>
+
+</details>
+</tool-use>
+
+---
+
+
+Updating `Process-DuplicateGroup` to delete duplicates instead of moving them:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "2666bd10-59d4-4d2b-be5e-cb4f437c6266"
+}
+```
+Status: **completed**
+
+
+</details>
+</tool-use>
+
+---
+
+
+Replacing the code that moves duplicates with code that deletes them:
+
+
+---
+
+<tool-use data-tool-type="unknown" data-tool-name="edit_file_v2">
+<details>
+<summary>Tool use: **edit_file_v2**</summary>
+
+
+Parameters:
+
+```json
+{
+  "relativeWorkspacePath": "a:\\1 Amazon_Active_Projects\\3 ProjectStructure\\organize_textures.ps1"
+}
+```
+Additional data:
+
+```json
+{
+  "codeblockId": "4847ba4f-4666-4d56-9eb5-fc4ea2aaabab"
+}
+```
+Status: **loading**
+
+
+</details>
+</tool-use>
+
+---
+
--- a/organize_textures.ps1
+++ b/organize_textures.ps1
@@ -130,7 +130,6 @@ function Process-DuplicateGroup {
    param(
        [array]$Files,
        [string]$CommonPath,
-        [string]$DuplicatesPath,
        [hashtable]$FilesInCommon,
        [switch]$StripPrefix,
        [string[]]$ValidPrefixes = @()
@@ -237,27 +236,14 @@ function Process-DuplicateGroup {
        Write-Warning "Failed to move first duplicate file: $($firstFile.FullName) - $($_.Exception.Message)"
    }
    
-    # Move remaining files to \common\duplicates with numbered suffixes
+    # Delete remaining duplicate files (they're replaced by the common file)
    for ($i = 1; $i -lt $Files.Count; $i++) {
        $fileObj = $Files[$i].File
-        $baseName = [System.IO.Path]::GetFileNameWithoutExtension($fileObj.Name)
-        $extension = [System.IO.Path]::GetExtension($fileObj.Name)
-        $numberedFileName = "${baseName}.$($i.ToString('000'))${extension}"
-        $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
-        
-        # Handle name conflicts in duplicates folder
-        $conflictCounter = 1
-        while (Test-Path -Path $duplicateDestinationPath) {
-            $numberedFileName = "${baseName}.$($i.ToString('000'))_${conflictCounter}${extension}"
-            $duplicateDestinationPath = Join-Path -Path $DuplicatesPath -ChildPath $numberedFileName
-            $conflictCounter++
-        }
-        
        try {
-            Move-Item -Path $fileObj.FullName -Destination $duplicateDestinationPath -Force
+            Remove-Item -Path $fileObj.FullName -Force
            $duplicateCount++
        } catch {
-            Write-Warning "Failed to move duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
+            Write-Warning "Failed to delete duplicate file: $($fileObj.FullName) - $($_.Exception.Message)"
        }
    }
    
@@ -307,18 +293,13 @@ if ($null -eq $blendfileFolders -or $blendfileFolders.Count -eq 0) {
        
        Write-Host "  Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Gray
        
-        # Create [blendfile]\common and [blendfile]\common\duplicates directories
+        # Create [blendfile]\common directory
        $blendfileCommonPath = Join-Path -Path $blendfileFolder.FullName -ChildPath "common"
-        $blendfileDuplicatesPath = Join-Path -Path $blendfileCommonPath -ChildPath "duplicates"
        
        if (-not (Test-Path -Path $blendfileCommonPath -PathType Container)) {
            New-Item -ItemType Directory -Path $blendfileCommonPath | Out-Null
        }
        
-        if (-not (Test-Path -Path $blendfileDuplicatesPath -PathType Container)) {
-            New-Item -ItemType Directory -Path $blendfileDuplicatesPath | Out-Null
-        }
-        
        # Track filenames already in [blendfile]\common
        $filesInBlendfileCommon = @{}
        
@@ -327,18 +308,18 @@ if ($null -eq $blendfileFolders -or $blendfileFolders.Count -eq 0) {
        $blendfileDuplicates = 0
        
        foreach ($group in $groupedByChecksum) {
-            $result = Process-DuplicateGroup -Files $group.Group -CommonPath $blendfileCommonPath -DuplicatesPath $blendfileDuplicatesPath -FilesInCommon $filesInBlendfileCommon
+            $result = Process-DuplicateGroup -Files $group.Group -CommonPath $blendfileCommonPath -FilesInCommon $filesInBlendfileCommon
            $blendfileMoved += $result.MovedCount
            $blendfileDuplicates += $result.DuplicateCount
        }
        
-        Write-Host "  Moved $blendfileMoved file(s) to \common, $blendfileDuplicates duplicate(s) to \common\duplicates" -ForegroundColor Green
+        Write-Host "  Moved $blendfileMoved file(s) to \common, deleted $blendfileDuplicates duplicate(s)" -ForegroundColor Green
        $totalPass1Moved += $blendfileMoved
        $totalPass1Duplicates += $blendfileDuplicates
    }
    
    Write-Host ""
-    Write-Host "Pass 1 complete: $totalPass1Moved file(s) moved, $totalPass1Duplicates duplicate(s) moved" -ForegroundColor Green
+    Write-Host "Pass 1 complete: $totalPass1Moved file(s) moved, $totalPass1Duplicates duplicate(s) deleted" -ForegroundColor Green
 }

 # ============================================================================
@@ -376,25 +357,19 @@ if ($null -eq $remainingFiles -or $remainingFiles.Count -eq 0) {
    
    Write-Host "Found $($groupedByChecksum.Count) unique checksums." -ForegroundColor Green
    
-    # Create \textures\common and \textures\common\duplicates directories
+    # Create \textures\common directory
    $rootCommonPath = Join-Path -Path $textureFolderPath -ChildPath "common"
-    $rootDuplicatesPath = Join-Path -Path $rootCommonPath -ChildPath "duplicates"
    
    if (-not (Test-Path -Path $rootCommonPath -PathType Container)) {
        New-Item -ItemType Directory -Path $rootCommonPath | Out-Null
        Write-Host "Created directory: $rootCommonPath" -ForegroundColor Green
    }
    
-    if (-not (Test-Path -Path $rootDuplicatesPath -PathType Container)) {
-        New-Item -ItemType Directory -Path $rootDuplicatesPath | Out-Null
-        Write-Host "Created directory: $rootDuplicatesPath" -ForegroundColor Green
-    }
-    
    # Track filenames already in \textures\common
    $filesInRootCommon = @{}
    
    # Process each checksum group
-    Write-Host "Moving files to \common and \common\duplicates..." -ForegroundColor Yellow
+    Write-Host "Moving files to \common and deleting duplicates..." -ForegroundColor Yellow
    $pass2Moved = 0
    $pass2Duplicates = 0
    
@@ -408,13 +383,13 @@ if ($null -eq $remainingFiles -or $remainingFiles.Count -eq 0) {
        
        # Multiple files with same checksum (duplicates across blendfiles)
        # Use StripPrefix to remove blendfile prefixes (e.g., "Demarco_", "Chan_") when all files share the same suffix
-        $result = Process-DuplicateGroup -Files $files -CommonPath $rootCommonPath -DuplicatesPath $rootDuplicatesPath -FilesInCommon $filesInRootCommon -StripPrefix -ValidPrefixes $validBlendfilePrefixes
+        $result = Process-DuplicateGroup -Files $files -CommonPath $rootCommonPath -FilesInCommon $filesInRootCommon -StripPrefix -ValidPrefixes $validBlendfilePrefixes
        $pass2Moved += $result.MovedCount
        $pass2Duplicates += $result.DuplicateCount
    }
    
    Write-Host ""
-    Write-Host "Pass 2 complete: $pass2Moved file(s) moved to \common, $pass2Duplicates duplicate(s) moved to \common\duplicates" -ForegroundColor Green
+    Write-Host "Pass 2 complete: $pass2Moved file(s) moved to \common, $pass2Duplicates duplicate(s) deleted" -ForegroundColor Green
 }

 Write-Host ""