diff --git a/src/main/java/com/sheepit/client/Client.java b/src/main/java/com/sheepit/client/Client.java index 03cf432..820ed32 100644 --- a/src/main/java/com/sheepit/client/Client.java +++ b/src/main/java/com/sheepit/client/Client.java @@ -24,11 +24,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.nio.file.FileSystemException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; @@ -66,6 +61,7 @@ import okhttp3.HttpUrl; @Data public class Client { public static final int MIN_JOB_ID = 20; //to distinguish between actual jobs and test frames private static final Locale LOCALE = Locale.ENGLISH; + private DirectoryManager directoryManager; private Gui gui; private Server server; private Configuration configuration; @@ -92,6 +88,7 @@ import okhttp3.HttpUrl; this.server = new Server(url_, this.configuration, this); this.log = Log.getInstance(this.configuration); this.gui = gui_; + this.directoryManager = new DirectoryManager(this.configuration, this.log); this.renderingJob = null; this.previousJob = null; this.jobsToValidate = new ArrayBlockingQueue(5); @@ -793,7 +790,7 @@ import okhttp3.HttpUrl; this.gui, this.log, String.format(LOCALE, "chunk %d/%d", i + 1, total), - ajob_.getRequiredProjectChunkPath(chunk.getId()), + this.directoryManager.getActualStoragePathFor(chunk), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()) ); @@ -811,7 +808,7 @@ import okhttp3.HttpUrl; this.gui, this.log, "renderer", - ajob.getRequiredRendererArchivePath(), + this.directoryManager.getActualStorageBinaryPathFor(ajob), ajob.getRendererMD5(), String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()) )).download(); @@ -823,15 +820,18 @@ import okhttp3.HttpUrl; protected int prepareWorkingDirectory(Job ajob) { int ret; - String bestRendererArchive = ajob.getRequiredRendererArchivePath(); - String renderer_archive = ajob.getRendererArchivePath(); + + String renderer_archive = this.directoryManager.getCacheBinaryPathFor(ajob); String renderer_path = ajob.getRendererDirectory(); File renderer_path_file = new File(renderer_path); - if (!new File(renderer_archive).exists()) { + // file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable) + if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) { this.gui.status("Copying renderer from shared downloads directory"); - this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + bestRendererArchive + " into " + renderer_archive); - copySharedChunk(bestRendererArchive, renderer_archive); + this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob)); + if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) { + log.error("Error while copying " + renderer_archive + " from shared downloads directory to working dir"); + } } if (!renderer_path_file.exists()) { @@ -862,11 +862,15 @@ import okhttp3.HttpUrl; String scene_path = ajob.getSceneDirectory(); File scene_path_file = new File(scene_path); + // chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile) for (Chunk chunk: ajob.getArchiveChunks()) { - if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) { + if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) { this.gui.status("Copying chunk from common directory"); - copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId())); + if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) { + this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir"); + } } + } /// download the chunks @@ -881,7 +885,7 @@ import okhttp3.HttpUrl; ret = Utils.unzipChunksIntoDirectory( - ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()), + ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), scene_path, ajob.getPassword(), log); @@ -895,28 +899,6 @@ import okhttp3.HttpUrl; return 0; } - private void copySharedChunk(String existingArchive, String targetArchive) { - Path existingArchivePath = Paths.get(existingArchive); - Path targetArchivePath = Paths.get(targetArchive); - try { - try { - Files.createLink(targetArchivePath, existingArchivePath); - log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath); - } - catch (UnsupportedOperationException // underlying file system does not support hard-linking - | FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported - | SecurityException // user is not allowed to create hard-links - ignore) { - // Creating hardlinks might not be supported on some filesystems - log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); - Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING); - } - } - catch (IOException e) { - this.gui.error("Error while copying " + existingArchive + " from shared downloads directory to working dir"); - } - } - protected Error.Type confirmJob(Job ajob, int checkpoint) { String url_real = String.format(LOCALE, "%s&rendertime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(), ajob.getProcessRender().getPeakMemoryUsed()); diff --git a/src/main/java/com/sheepit/client/DirectoryManager.java b/src/main/java/com/sheepit/client/DirectoryManager.java new file mode 100644 index 0000000..6bdb9bc --- /dev/null +++ b/src/main/java/com/sheepit/client/DirectoryManager.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2023 Laurent CLOUET + * Author Laurent CLOUET + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package com.sheepit.client; + +import com.sheepit.client.datamodel.Chunk; +import lombok.AllArgsConstructor; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystemException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +@AllArgsConstructor +public class DirectoryManager { + private Configuration configuration; + private Log log; + + public String getActualStoragePathFor(Chunk chunk) { + return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk); + } + + public String getActualStorageBinaryPathFor(Job job) { + return isSharedEnabled() ? getSharedBinaryPathFor(job) : getCacheBinaryPathFor(job); + } + + public String getCachePathFor(Chunk chunk) { + return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool"; + } + + public String getSharedPathFor(Chunk chunk) { + return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool"; + } + + public String getCacheBinaryPathFor(Job job) { + return configuration.getStorageDir().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip"; + } + + public String getSharedBinaryPathFor(Job job) { + return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip"; + } + + public boolean isSharedEnabled() { + return configuration.getSharedDownloadsDirectory() != null && configuration.getSharedDownloadsDirectory().exists(); + } + + public boolean copyBinaryFromSharedToCache(Job job) { + return copyFileFromSharedToCache(getSharedBinaryPathFor(job), getCacheBinaryPathFor(job)); + } + + public boolean copyChunkFromSharedToCache(Chunk chunk) { + return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk)); + } + + private boolean copyFileFromSharedToCache(String source, String target) { + Path existingArchivePath = Paths.get(source); + Path targetArchivePath = Paths.get(target); + + if (existingArchivePath.equals(targetArchivePath)) { + // target are the same, do nothing + return true; + } + + try { + try { + Files.createLink(targetArchivePath, existingArchivePath); + log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath); + } + catch (UnsupportedOperationException // underlying file system does not support hard-linking + | FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported + | SecurityException // user is not allowed to create hard-links + ignore) { + // Creating hardlinks might not be supported on some filesystems + log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); + Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING); + } + } + catch (IOException e) { + log.error("Error while copying " + source + " from shared downloads directory to working dir"); + return false; + } + + return true; + } +} diff --git a/src/main/java/com/sheepit/client/Job.java b/src/main/java/com/sheepit/client/Job.java index 237f266..2d7947d 100644 --- a/src/main/java/com/sheepit/client/Job.java +++ b/src/main/java/com/sheepit/client/Job.java @@ -147,32 +147,10 @@ import java.util.regex.Pattern; return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererMD5; } - public String getRequiredRendererArchivePath() { - if (configuration.getSharedDownloadsDirectory() != null) { - return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + rendererMD5 + ".zip"; - } - else { - return getRendererArchivePath(); - } - } - public String getRendererPath() { return getRendererDirectory() + File.separator + OS.getOS().getRenderBinaryPath(); } - public String getRendererArchivePath() { - return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip"; - } - - public String getRequiredProjectChunkPath(String chunk) { - if (configuration.getSharedDownloadsDirectory() != null) { - return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk + ".wool"; - } - else { - return getSceneArchiveChunkPath(chunk); - } - } - public String getSceneDirectory() { return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id; } @@ -181,10 +159,6 @@ import java.util.regex.Pattern; return getSceneDirectory() + File.separator + this.path; } - public String getSceneArchiveChunkPath(String chunk) { - return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk + ".wool"; - } - public Error.Type render(Observer renderStarted) { gui.status("Rendering"); RenderProcess process = getProcessRender();