From f1dafd5d8e2220c19db7df67ed3b2f5e80acc625 Mon Sep 17 00:00:00 2001 From: Grische Date: Fri, 18 Feb 2022 02:20:57 +0100 Subject: [PATCH 1/2] Try to hardlink shared archives before copying This will reduce both the disk usage and IO load on systems with multiple clients. --- src/com/sheepit/client/Client.java | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/com/sheepit/client/Client.java b/src/com/sheepit/client/Client.java index 00d409a..1f5f339 100644 --- a/src/com/sheepit/client/Client.java +++ b/src/com/sheepit/client/Client.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; @@ -950,12 +951,7 @@ import okhttp3.HttpUrl; if (!new File(renderer_archive).exists()) { this.gui.status("Copying renderer from shared downloads directory"); - try { - Files.copy(Paths.get(bestRendererArchive), Paths.get(renderer_archive), StandardCopyOption.REPLACE_EXISTING); - } - catch (IOException e) { - this.gui.error("Error while copying renderer from shared downloads directory to working dir"); - } + copySharedArchive(bestRendererArchive, renderer_archive); } if (!renderer_path_file.exists()) { @@ -989,13 +985,7 @@ import okhttp3.HttpUrl; if (!new File(scene_archive).exists()) { this.gui.status("Copying scene from common directory"); - - try { - Files.copy(Paths.get(bestSceneArchive), Paths.get(scene_archive), StandardCopyOption.REPLACE_EXISTING); - } - catch (IOException e) { - this.gui.error("Error while copying scene from common directory to working dir"); - } + copySharedArchive(bestSceneArchive, scene_archive); } if (!scene_path_file.exists()) { @@ -1018,6 +1008,25 @@ import okhttp3.HttpUrl; return 0; } + private void copySharedArchive(String existingArchive, String targetArchive) { + Path existingArchivePath = Paths.get(existingArchive); + Path targetArchivePath = Paths.get(targetArchive); + try { + try { + Files.createLink(targetArchivePath, existingArchivePath); + log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath); + } + catch (UnsupportedOperationException x) { + // Creating hardlinks might not be supported on some filesystems + log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); + Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING); + } + } + catch (IOException e) { + this.gui.error("Error while copying " + existingArchive + " from shared downloads directory to working dir"); + } + } + protected Error.Type confirmJob(Job ajob, int checkpoint) { String url_real = String.format(LOCALE, "%s&rendertime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getDuration(), ajob.getProcessRender().getPeakMemoryUsed()); From 84a45a323977d6239f1d64e997a9295beee18f13 Mon Sep 17 00:00:00 2001 From: Grische Date: Sun, 27 Feb 2022 20:46:58 +0100 Subject: [PATCH 2/2] Handle all edge cases for hard-linking archives --- src/com/sheepit/client/Client.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/com/sheepit/client/Client.java b/src/com/sheepit/client/Client.java index 1f5f339..89068f3 100644 --- a/src/com/sheepit/client/Client.java +++ b/src/com/sheepit/client/Client.java @@ -24,6 +24,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -1007,7 +1008,7 @@ import okhttp3.HttpUrl; return 0; } - + private void copySharedArchive(String existingArchive, String targetArchive) { Path existingArchivePath = Paths.get(existingArchive); Path targetArchivePath = Paths.get(targetArchive); @@ -1016,7 +1017,10 @@ import okhttp3.HttpUrl; Files.createLink(targetArchivePath, existingArchivePath); log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath); } - catch (UnsupportedOperationException x) { + catch (UnsupportedOperationException // underlying file system does not support hard-linking + | FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported + | SecurityException // user is not allowed to create hard-links + ignore) { // Creating hardlinks might not be supported on some filesystems log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);