From 94b36241b0a0d4990f67a58f3fe17e54efea7e06 Mon Sep 17 00:00:00 2001 From: Sheepit Renderfarm Date: Wed, 20 Nov 2024 13:05:20 +0000 Subject: [PATCH] Feat: use chunk for renderer --- src/main/java/com/sheepit/client/Client.java | 61 ++++++------------- .../com/sheepit/client/Configuration.java | 6 +- .../com/sheepit/client/DirectoryManager.java | 16 ----- src/main/java/com/sheepit/client/Job.java | 4 +- src/main/java/com/sheepit/client/Server.java | 5 +- src/main/java/com/sheepit/client/Utils.java | 9 +++ .../client/datamodel/RendererInfos.java | 4 ++ 7 files changed, 41 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/sheepit/client/Client.java b/src/main/java/com/sheepit/client/Client.java index 58bdfaf..5dda700 100644 --- a/src/main/java/com/sheepit/client/Client.java +++ b/src/main/java/com/sheepit/client/Client.java @@ -796,18 +796,15 @@ import okhttp3.HttpUrl; return err; } - protected Error.Type downloadSceneFile(Job ajob) throws SheepItException { - int total = ajob.getArchiveChunks().size(); - int threads = Math.max(1, Math.min(total, 12)); // at least one thread, to avoid IllegalArgumentException if total = 0 + private Error.Type downloadChunks(List chunks, String status) throws SheepItException { + int threads = Math.max(1, Math.min(chunks.size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0 ExecutorService executor = Executors.newFixedThreadPool(threads); List> tasks = new ArrayList<>(); - this.gui.getDownloadProgress().reset("Downloading project"); + this.gui.getDownloadProgress().reset(status); - for (int i = 0; i < total; i++) { - Chunk chunk = ajob.getArchiveChunks().get(i); - + for (Chunk chunk : chunks) { Callable downloadTask = () -> { DownloadManager downloadManager = new DownloadManager( this.server, @@ -844,16 +841,12 @@ import okhttp3.HttpUrl; return Type.OK; } + protected Error.Type downloadSceneFile(Job ajob) throws SheepItException { + return this.downloadChunks(ajob.getArchiveChunks(), "Downloading Project"); + } + protected Error.Type downloadExecutable(Job ajob) throws SheepItException { - this.gui.getDownloadProgress().reset("Downloading Blender"); - return (new DownloadManager( - this.server, - this.gui, - this.log, - this.directoryManager.getActualStorageBinaryPathFor(ajob), - ajob.getRendererMD5(), - String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()) - )).download(); + return this.downloadChunks(ajob.getRendererChunks(), "Downloading Blender"); } protected void removeSceneDirectory(Job ajob) { @@ -863,17 +856,18 @@ import okhttp3.HttpUrl; protected int prepareWorkingDirectory(Job ajob) { int ret; - String rendererArchive = this.directoryManager.getCacheBinaryPathFor(ajob); String rendererPath = ajob.getRendererDirectory(); File rendererPathFile = new File(rendererPath); - // file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable) - if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) { - this.gui.status("Copying renderer from shared downloads directory"); - this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob)); - if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) { - log.error("Error while copying " + rendererArchive + " from shared downloads directory to working dir"); + // chunk files are already downloaded, either on shared directory or cache directory + for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererChunks(), ajob.getArchiveChunks())) { + if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) { + this.gui.status("Copying chunk from common directory"); + if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) { + this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir"); + } } + } if (!rendererPathFile.exists()) { @@ -881,14 +875,12 @@ import okhttp3.HttpUrl; rendererPathFile.mkdir(); this.gui.status("Extracting renderer"); - this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + rendererArchive + " into " + rendererPath); + this.log.debug("Client::prepareWorkingDirectory Extracting renderer into " + rendererPath); // unzip the archive - ret = Utils.unzipFileIntoDirectory(rendererArchive, rendererPath, null, log); + ret = Utils.unzipChunksIntoDirectory(ajob.getRendererChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log); if (ret != 0) { - this.log.error( - "Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + rendererArchive + ", " + rendererPath + ") returned " - + ret); + this.log.error("Client::prepareWorkingDirectory, error(1) with Utils.unzipChunksIntoDirectory(" + rendererPath + ") returned " + ret); this.gui.error(String.format("Unable to extract the renderer (error %d)", ret)); return -1; } @@ -904,19 +896,6 @@ import okhttp3.HttpUrl; String scenePath = ajob.getSceneDirectory(); File scenePathFile = new File(scenePath); - // chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile) - for (Chunk chunk: ajob.getArchiveChunks()) { - if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) { - this.gui.status("Copying chunk from common directory"); - if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) { - this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir"); - } - } - - } - - /// download the chunks - if (!scenePathFile.exists()) { // we create the directory scenePathFile.mkdir(); diff --git a/src/main/java/com/sheepit/client/Configuration.java b/src/main/java/com/sheepit/client/Configuration.java index 36ee581..d781517 100644 --- a/src/main/java/com/sheepit/client/Configuration.java +++ b/src/main/java/com/sheepit/client/Configuration.java @@ -57,7 +57,7 @@ import lombok.Data; private String logDirectory; private File workingDirectory; private File sharedDownloadsDirectory; - private File woolCacheDirectory; // store all the chunks (and binary for now) + private File woolCacheDirectory; // store all the chunks (project and blender) private boolean userHasSpecifiedACacheDir; private String login; private String password; @@ -273,7 +273,7 @@ import lombok.Data; try { String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase(); String name = file.getName().substring(0, file.getName().length() - 1 * extension.length()); - if (".zip".equals(extension) || ".wool".equals(extension)) { + if (".wool".equals(extension)) { // check if the md5 of the file is ok String md5_local = Utils.md5(file.getAbsolutePath()); @@ -339,7 +339,7 @@ import lombok.Data; try { String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase(); String name = file.getName().substring(0, file.getName().length() - 1 * extension.length()); - if (".zip".equals(extension) || ".wool".equals(extension)) { + if (".wool".equals(extension)) { // check if the md5 of the file is ok String md5_local = Utils.md5(file.getAbsolutePath()); diff --git a/src/main/java/com/sheepit/client/DirectoryManager.java b/src/main/java/com/sheepit/client/DirectoryManager.java index f705085..2e160ba 100644 --- a/src/main/java/com/sheepit/client/DirectoryManager.java +++ b/src/main/java/com/sheepit/client/DirectoryManager.java @@ -39,10 +39,6 @@ public class DirectoryManager { return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk); } - public String getActualStorageBinaryPathFor(Job job) { - return isSharedEnabled() ? getSharedBinaryPathFor(job) : getCacheBinaryPathFor(job); - } - public String getCachePathFor(Chunk chunk) { return configuration.getWoolCacheDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool"; } @@ -51,22 +47,10 @@ public class DirectoryManager { return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool"; } - public String getCacheBinaryPathFor(Job job) { - return configuration.getWoolCacheDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip"; - } - - public String getSharedBinaryPathFor(Job job) { - return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip"; - } - public boolean isSharedEnabled() { return configuration.getSharedDownloadsDirectory() != null && configuration.getSharedDownloadsDirectory().exists(); } - public boolean copyBinaryFromSharedToCache(Job job) { - return copyFileFromSharedToCache(getSharedBinaryPathFor(job), getCacheBinaryPathFor(job)); - } - public boolean copyChunkFromSharedToCache(Chunk chunk) { return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk)); } diff --git a/src/main/java/com/sheepit/client/Job.java b/src/main/java/com/sheepit/client/Job.java index 34e5acb..86382f7 100644 --- a/src/main/java/com/sheepit/client/Job.java +++ b/src/main/java/com/sheepit/client/Job.java @@ -65,6 +65,7 @@ import java.util.regex.Pattern; private String frameNumber; private List archiveChunks; private String rendererMD5; + private List rendererChunks; private String id; private String outputImagePath; @@ -92,7 +93,7 @@ import java.util.regex.Pattern; private Log log; public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_, - String script_, List archiveChunks_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_, + String script_, List archiveChunks_, String rendererMd5_, List rendererChunks_, String name_, char[] password_, boolean synchronous_upload_, String update_method_) { configuration = config_; id = id_; @@ -103,6 +104,7 @@ import java.util.regex.Pattern; validationUrl = validationUrl_; archiveChunks = archiveChunks_; rendererMD5 = rendererMd5_; + rendererChunks = rendererChunks_; name = name_; password = password_.clone(); synchronousUpload = synchronous_upload_; diff --git a/src/main/java/com/sheepit/client/Server.java b/src/main/java/com/sheepit/client/Server.java index 99102b9..53559e7 100644 --- a/src/main/java/com/sheepit/client/Server.java +++ b/src/main/java/com/sheepit/client/Server.java @@ -400,7 +400,7 @@ public class Server extends Thread { jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator), jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl, jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(), - jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(), + jobData.getRenderTask().getRendererInfos().getChunks(), jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(), "1".equals(jobData.getRenderTask().getSynchronousUpload()), jobData.getRenderTask().getRendererInfos().getUpdateMethod()); } catch (SheepItException e) { @@ -688,7 +688,7 @@ public class Server extends Thread { try { String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase(); String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length()); - if (".zip".equals(extension) || ".wool".equals(extension)) { + if (".wool".equals(extension)) { // node_file.setAttribute("md5", name); FileMD5 fileMD5 = new FileMD5(); fileMD5.setMd5(name); @@ -728,7 +728,6 @@ public class Server extends Thread { for(String path: paths) { new File(path + ".wool").delete(); - new File(path + ".zip").delete(); Utils.delete(new File(path)); } diff --git a/src/main/java/com/sheepit/client/Utils.java b/src/main/java/com/sheepit/client/Utils.java index 739804c..81d262f 100644 --- a/src/main/java/com/sheepit/client/Utils.java +++ b/src/main/java/com/sheepit/client/Utils.java @@ -35,6 +35,7 @@ import java.io.StringWriter; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.HashMap; @@ -393,4 +394,12 @@ public class Utils { } return true; } + + public static List concatWithCollection(List list1, List list2) { + List resultList = new ArrayList<>(list1.size() + list1.size()); + resultList.addAll(list1); + resultList.addAll(list2); + + return resultList; + } } diff --git a/src/main/java/com/sheepit/client/datamodel/RendererInfos.java b/src/main/java/com/sheepit/client/datamodel/RendererInfos.java index 6486c8e..10b4f67 100644 --- a/src/main/java/com/sheepit/client/datamodel/RendererInfos.java +++ b/src/main/java/com/sheepit/client/datamodel/RendererInfos.java @@ -4,8 +4,11 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; import org.simpleframework.xml.Attribute; +import org.simpleframework.xml.ElementList; import org.simpleframework.xml.Root; +import java.util.List; + @NoArgsConstructor @Root(strict = false, name = "renderer") @ToString public class RendererInfos { @Attribute(name = "md5") @Getter private String md5; @@ -14,4 +17,5 @@ import org.simpleframework.xml.Root; @Attribute(name = "update_method") @Getter private String updateMethod; + @ElementList(name = "chunks") @Getter private List chunks; }