diff --git a/src/main/java/com/sheepit/client/Client.java b/src/main/java/com/sheepit/client/Client.java index 5f6c6b5..ac411c0 100644 --- a/src/main/java/com/sheepit/client/Client.java +++ b/src/main/java/com/sheepit/client/Client.java @@ -44,9 +44,11 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.sheepit.client.Error.ServerCode; import com.sheepit.client.Error.Type; +import com.sheepit.client.datamodel.Chunk; import com.sheepit.client.exception.SheepItException; import com.sheepit.client.exception.SheepItExceptionBadResponseFromServer; import com.sheepit.client.exception.SheepItExceptionNoRendererAvailable; @@ -859,13 +861,18 @@ import okhttp3.HttpUrl; } protected Error.Type downloadSceneFile(Job ajob_) throws SheepItException { - return this.downloadFile(ajob_, ajob_.getRequiredSceneArchivePath(), ajob_.getSceneMD5(), - String.format(LOCALE, "%s?type=job&job=%s", this.server.getPage("download-archive"), ajob_.getId()), "project"); + for (Chunk chunk : ajob_.getArchiveChunks()) { + Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), "project"); + if (ret != Type.OK) { + return ret; + } + } + return Type.OK; } protected Error.Type downloadExecutable(Job ajob) throws SheepItException { return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(), - String.format(LOCALE, "%s?type=binary&job=%s", this.server.getPage("download-archive"), ajob.getId()), "renderer"); + String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer"); } private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException { @@ -900,7 +907,7 @@ import okhttp3.HttpUrl; } } else { - // The file doesn't yet existing not is being downloaded by another client, so immediately create the file with zero bytes to allow early + // The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early // detection by other concurrent clients and start downloading process try { File file = new File(local_path + ".partial"); @@ -1008,7 +1015,7 @@ import okhttp3.HttpUrl; if (!new File(renderer_archive).exists()) { this.gui.status("Copying renderer from shared downloads directory"); - copySharedArchive(bestRendererArchive, renderer_archive); + copySharedChunk(bestRendererArchive, renderer_archive); } if (!renderer_path_file.exists()) { @@ -1035,16 +1042,18 @@ import okhttp3.HttpUrl; } } - String bestSceneArchive = ajob.getRequiredSceneArchivePath(); - String scene_archive = ajob.getSceneArchivePath(); String scene_path = ajob.getSceneDirectory(); File scene_path_file = new File(scene_path); - if (!new File(scene_archive).exists()) { - this.gui.status("Copying scene from common directory"); - copySharedArchive(bestSceneArchive, scene_archive); + for (Chunk chunk: ajob.getArchiveChunks()) { + if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) { + this.gui.status("Copying chunk from common directory"); + copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId())); + } } + /// download the chunks + if (!scene_path_file.exists()) { // we create the directory scene_path_file.mkdir(); @@ -1052,11 +1061,15 @@ import okhttp3.HttpUrl; this.gui.status("Extracting project"); // unzip the archive - ret = Utils.unzipFileIntoDirectory(scene_archive, scene_path, ajob.getPassword(), log); + + + ret = Utils.unzipChunksIntoDirectory( + ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()), + scene_path, + ajob.getPassword(), + log); if (ret != 0) { - this.log.error( - "Client::prepareWorkingDirectory, error(2) with Utils.unzipFileIntoDirectory(" + scene_archive + ", " + scene_path + ") returned " - + ret); + this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret); this.gui.error(String.format("Unable to extract the scene (error %d)", ret)); return -2; } @@ -1065,7 +1078,7 @@ import okhttp3.HttpUrl; return 0; } - private void copySharedArchive(String existingArchive, String targetArchive) { + private void copySharedChunk(String existingArchive, String targetArchive) { Path existingArchivePath = Paths.get(existingArchive); Path targetArchivePath = Paths.get(targetArchive); try { diff --git a/src/main/java/com/sheepit/client/Configuration.java b/src/main/java/com/sheepit/client/Configuration.java index bdd1cdc..0d822a4 100644 --- a/src/main/java/com/sheepit/client/Configuration.java +++ b/src/main/java/com/sheepit/client/Configuration.java @@ -258,7 +258,7 @@ import lombok.Data; try { String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase(); String name = file.getName().substring(0, file.getName().length() - 1 * extension.length()); - if (extension.equals(".zip")) { + if (extension.equals(".zip") || extension.equals(".wool")) { // check if the md5 of the file is ok String md5_local = Utils.md5(file.getAbsolutePath()); diff --git a/src/main/java/com/sheepit/client/Job.java b/src/main/java/com/sheepit/client/Job.java index 24345a2..9384efa 100644 --- a/src/main/java/com/sheepit/client/Job.java +++ b/src/main/java/com/sheepit/client/Job.java @@ -21,6 +21,7 @@ package com.sheepit.client; import com.sheepit.client.Configuration.ComputeType; import com.sheepit.client.Error.Type; +import com.sheepit.client.datamodel.Chunk; import com.sheepit.client.os.OS; import lombok.Data; import lombok.Getter; @@ -62,7 +63,7 @@ import java.util.regex.Pattern; public static final int SHOW_BASE_ICON = -1; private String frameNumber; - private String sceneMD5; + private List archiveChunks; private String rendererMD5; private String id; private String outputImagePath; @@ -90,7 +91,7 @@ import java.util.regex.Pattern; private Log log; public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_, - String script_, String sceneMd5_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_, + String script_, List archiveChunks_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_, String update_method_) { configuration = config_; id = id_; @@ -99,7 +100,7 @@ import java.util.regex.Pattern; useGPU = use_gpu; rendererCommand = command_; validationUrl = validationUrl_; - sceneMD5 = sceneMd5_; + archiveChunks = archiveChunks_; rendererMD5 = rendererMd5_; name = name_; password = password_; @@ -134,8 +135,8 @@ import java.util.regex.Pattern; public String toString() { return String - .format("Job (numFrame '%s' sceneMD5 '%s' rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)", - frameNumber, sceneMD5, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render); + .format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)", + frameNumber, archiveChunks, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render); } public String getPrefixOutputImage() { @@ -163,25 +164,25 @@ import java.util.regex.Pattern; return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip"; } - public String getRequiredSceneArchivePath() { + public String getRequiredProjectChunkPath(String chunk) { if (configuration.getSharedDownloadsDirectory() != null) { - return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip"; + return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk + ".wool"; } else { - return getSceneArchivePath(); + return getSceneArchiveChunkPath(chunk); } } public String getSceneDirectory() { - return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5; + return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id; } public String getScenePath() { return getSceneDirectory() + File.separator + this.path; } - public String getSceneArchivePath() { - return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip"; + public String getSceneArchiveChunkPath(String chunk) { + return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk + ".wool"; } public Error.Type render(Observer renderStarted) { diff --git a/src/main/java/com/sheepit/client/Server.java b/src/main/java/com/sheepit/client/Server.java index 2735d41..4466396 100644 --- a/src/main/java/com/sheepit/client/Server.java +++ b/src/main/java/com/sheepit/client/Server.java @@ -401,7 +401,7 @@ public class Server extends Thread { return new Job(this.user_config, this.client.getGui(), this.client.getLog(), jobData.getRenderTask().getId(), jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator), jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl, - jobData.getRenderTask().getScript(), jobData.getRenderTask().getArchive_md5(), jobData.getRenderTask().getRendererInfos().getMd5(), + jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(), jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(), jobData.getRenderTask().getSynchronous_upload().equals("1"), jobData.getRenderTask().getRendererInfos().getUpdate_method()); } @@ -491,6 +491,7 @@ public class Server extends Thread { } public Error.Type HTTPGetFile(String url_, String destination_, Gui gui_, String status_) throws SheepItException { + this.log.debug("Server::HTTPGetFile destination: " + destination_); InputStream is = null; OutputStream output = null; @@ -694,7 +695,7 @@ public class Server extends Thread { try { String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase(); String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length()); - if (extension.equals(".zip")) { + if (extension.equals(".zip") || extension.equals(".wool")) { // node_file.setAttribute("md5", name); FileMD5 fileMD5 = new FileMD5(); fileMD5.setMd5(name); @@ -724,24 +725,20 @@ public class Server extends Thread { if (fileMD5s != null && fileMD5s.isEmpty() == false) { for (FileMD5 fileMD5 : fileMD5s) { if ("delete".equals(fileMD5.getAction()) && fileMD5.getMd5() != null && fileMD5.getMd5().isEmpty() == false) { - String path = this.user_config.getWorkingDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5(); - this.log.debug("Server::handleFileMD5DeleteDocument delete old file " + path); - File file_to_delete = new File(path + ".zip"); - file_to_delete.delete(); - Utils.delete(new File(path)); - - //also delete in binary cache - path = this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5(); - file_to_delete = new File(path + ".zip"); - file_to_delete.delete(); - Utils.delete(new File(path)); + List paths = new ArrayList<>(); + paths.add(this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5()); //also delete in binary cache + paths.add(this.user_config.getWorkingDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5()); // If we are using a shared downloads directory, then delete the file from the shared downloads directory as well :) if (this.user_config.getSharedDownloadsDirectory() != null) { - String commonCacheFile = this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5(); - this.log.debug("Server::handleFileMD5DeleteDocument delete common file " + commonCacheFile + ".zip"); - file_to_delete = new File(commonCacheFile + ".zip"); - file_to_delete.delete(); + paths.add(this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5()); + } + + for(String path: paths) { + new File(path + ".wool").delete(); + new File(path + ".zip").delete(); + + Utils.delete(new File(path)); } } } diff --git a/src/main/java/com/sheepit/client/Utils.java b/src/main/java/com/sheepit/client/Utils.java index cf0b937..6c3e129 100644 --- a/src/main/java/com/sheepit/client/Utils.java +++ b/src/main/java/com/sheepit/client/Utils.java @@ -22,13 +22,18 @@ package com.sheepit.client; import com.sheepit.client.Error.ServerCode; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.io.inputstream.ZipInputStream; +import net.lingala.zip4j.model.LocalFileHeader; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; @@ -36,12 +41,10 @@ import java.io.StringWriter; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Calendar; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.regex.Matcher; @@ -77,6 +80,64 @@ public class Utils { return 0; } + public static int unzipChunksIntoDirectory(List full_path_chunks, String destinationDirectory, char[] password, Log log) { + try { + // STEP 1: assemble the chunks into an actual zip (in RAM) + ByteArrayOutputStream unzippedData = new ByteArrayOutputStream(); + + for (String full_path_chunk: full_path_chunks) { + byte[] data = Files.readAllBytes(Paths.get(full_path_chunk)); + + unzippedData.write(data); + } + byte[] full_data = unzippedData.toByteArray(); + + // STEP 2: unzip the zip like before + ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(full_data)); + if (password != null) { + zipInputStream.setPassword(password); + } + LocalFileHeader fileHeader = null; + while ((fileHeader = zipInputStream.getNextEntry()) != null) { + String outFilePath = destinationDirectory + File.separator + fileHeader.getFileName(); + File outFile = new File(outFilePath); + + //Checks if the file is a directory + if (fileHeader.isDirectory()) { + outFile.mkdirs(); + continue; + } + + File parentDir = outFile.getParentFile(); + if (parentDir.exists() == false) { + parentDir.mkdirs(); + } + + FileOutputStream os = new FileOutputStream(outFile); + + int readLen = -1; + byte[] buff = new byte[1024]; + + //Loop until End of File and write the contents to the output stream + while ((readLen = zipInputStream.read(buff)) != -1) { + os.write(buff, 0, readLen); + } + + os.close(); + } + zipInputStream.close(); + } + catch (IOException e) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + log.debug("Utils::unzipChunksIntoDirectory exception " + e + " stacktrace: " + sw.toString()); + return -1; + } + + return 0; + } + public static String md5(String path_of_file_) { Md5 md5 = new Md5(); return md5.get(path_of_file_); diff --git a/src/main/java/com/sheepit/client/datamodel/Chunk.java b/src/main/java/com/sheepit/client/datamodel/Chunk.java new file mode 100644 index 0000000..22d5588 --- /dev/null +++ b/src/main/java/com/sheepit/client/datamodel/Chunk.java @@ -0,0 +1,16 @@ +package com.sheepit.client.datamodel; + +import lombok.Data; +import lombok.ToString; +import org.simpleframework.xml.Attribute; +import org.simpleframework.xml.Root; + +@Root(strict = false, name = "chunk") @Data @ToString public class Chunk { + + @Attribute private String md5; + + @Attribute private String id; + + public Chunk() { + } +} diff --git a/src/main/java/com/sheepit/client/datamodel/RenderTask.java b/src/main/java/com/sheepit/client/datamodel/RenderTask.java index 8057f36..ce70286 100644 --- a/src/main/java/com/sheepit/client/datamodel/RenderTask.java +++ b/src/main/java/com/sheepit/client/datamodel/RenderTask.java @@ -4,15 +4,18 @@ import lombok.Getter; import lombok.ToString; import org.simpleframework.xml.Attribute; import org.simpleframework.xml.Element; +import org.simpleframework.xml.ElementList; import org.simpleframework.xml.Root; +import java.util.List; + @Root(strict = false, name = "job") @Getter @ToString public class RenderTask { @Attribute(name = "id") private String id; @Attribute(name = "use_gpu") private int useGpu; - @Attribute(name = "archive_md5") private String archive_md5; + @ElementList(name = "chunks") private List chunks; @Attribute(name = "path") private String path;