Feat: use chunk for renderer

src/main/java/com/sheepit/client/Client.java

@@ -796,18 +796,15 @@ import okhttp3.HttpUrl;
 		return err;
 	}
 	
-	protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
+	private Error.Type downloadChunks(List<Chunk> chunks, String status) throws SheepItException {
-		int total = ajob.getArchiveChunks().size();
+		int threads = Math.max(1, Math.min(chunks.size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
-		int threads = Math.max(1, Math.min(total, 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
 		
 		ExecutorService executor = Executors.newFixedThreadPool(threads);
 		List<Callable<Error.Type>> tasks = new ArrayList<>();
 		
-		this.gui.getDownloadProgress().reset("Downloading project");
+		this.gui.getDownloadProgress().reset(status);
-		
-		for (int i = 0; i < total; i++) {
-			Chunk chunk = ajob.getArchiveChunks().get(i);
 		
+		for (Chunk chunk : chunks) {
 			Callable<Type> downloadTask = () -> {
 				DownloadManager downloadManager  = new DownloadManager(
 					this.server,
@@ -844,16 +841,12 @@ import okhttp3.HttpUrl;
 		return Type.OK;
 	}
 	
+	protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
+		return this.downloadChunks(ajob.getArchiveChunks(), "Downloading Project");
+	}
+	
 	protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
-		this.gui.getDownloadProgress().reset("Downloading Blender");
+		return this.downloadChunks(ajob.getRendererChunks(), "Downloading Blender");
-		return (new DownloadManager(
-			this.server,
-			this.gui,
-			this.log,
-			this.directoryManager.getActualStorageBinaryPathFor(ajob),
-			ajob.getRendererMD5(),
-			String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
-		)).download();
 	}
 	
 	protected void removeSceneDirectory(Job ajob) {
@@ -863,17 +856,18 @@ import okhttp3.HttpUrl;
 	protected int prepareWorkingDirectory(Job ajob) {
 		int ret;
 		
-		String rendererArchive = this.directoryManager.getCacheBinaryPathFor(ajob);
 		String rendererPath = ajob.getRendererDirectory();
 		File rendererPathFile = new File(rendererPath);
 		
-		// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
+		// chunk files are already downloaded, either on shared directory or cache directory
-		if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
+		for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererChunks(), ajob.getArchiveChunks())) {
-			this.gui.status("Copying renderer from shared downloads directory");
+			if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
-			this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
+				this.gui.status("Copying chunk from common directory");
-			if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
+				if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
-				log.error("Error while copying " + rendererArchive + " from shared downloads directory to working dir");
+					this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
+				}
 			}
+			
 		}
 		
 		if (!rendererPathFile.exists()) {
@@ -881,14 +875,12 @@ import okhttp3.HttpUrl;
 			rendererPathFile.mkdir();
 			
 			this.gui.status("Extracting renderer");
-			this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + rendererArchive + " into " + rendererPath);
+			this.log.debug("Client::prepareWorkingDirectory Extracting renderer into " + rendererPath);
 			
 			// unzip the archive
-			ret = Utils.unzipFileIntoDirectory(rendererArchive, rendererPath, null, log);
+			ret = Utils.unzipChunksIntoDirectory(ajob.getRendererChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
 			if (ret != 0) {
-				this.log.error(
+				this.log.error("Client::prepareWorkingDirectory, error(1) with Utils.unzipChunksIntoDirectory(" + rendererPath + ") returned " + ret);
-						"Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + rendererArchive + ", " + rendererPath + ") returned "
-								+ ret);
 				this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
 				return -1;
 			}
@@ -904,19 +896,6 @@ import okhttp3.HttpUrl;
 		String scenePath = ajob.getSceneDirectory();
 		File scenePathFile = new File(scenePath);
 		
-		// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
-		for (Chunk chunk: ajob.getArchiveChunks()) {
-			if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
-				this.gui.status("Copying chunk from common directory");
-				if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
-					this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
-				}
-			}
-			
-		}
-		
-		/// download the chunks
-		
 		if (!scenePathFile.exists()) {
 			// we create the directory
 			scenePathFile.mkdir();

src/main/java/com/sheepit/client/Configuration.java

@@ -57,7 +57,7 @@ import lombok.Data;
 	private String logDirectory;
 	private File workingDirectory;
 	private File sharedDownloadsDirectory;
-	private File woolCacheDirectory; // store all the chunks (and binary for now)
+	private File woolCacheDirectory; // store all the chunks (project and blender)
 	private boolean userHasSpecifiedACacheDir;
 	private String login;
 	private String password;
@@ -273,7 +273,7 @@ import lombok.Data;
 					try {
 						String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
 						String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
-						if (".zip".equals(extension) || ".wool".equals(extension)) {
+						if (".wool".equals(extension)) {
 							// check if the md5 of the file is ok
 							String md5_local = Utils.md5(file.getAbsolutePath());
 							
@@ -339,7 +339,7 @@ import lombok.Data;
 				try {
 					String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
 					String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
-					if (".zip".equals(extension) || ".wool".equals(extension)) {
+					if (".wool".equals(extension)) {
 						// check if the md5 of the file is ok
 						String md5_local = Utils.md5(file.getAbsolutePath());
 						

src/main/java/com/sheepit/client/DirectoryManager.java

@@ -39,10 +39,6 @@ public class DirectoryManager {
 		return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk);
 	}
 	
-	public String getActualStorageBinaryPathFor(Job job) {
-		return isSharedEnabled() ? getSharedBinaryPathFor(job) : getCacheBinaryPathFor(job);
-	}
-	
 	public String getCachePathFor(Chunk chunk) {
 		return configuration.getWoolCacheDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool";
 	}
@@ -51,22 +47,10 @@ public class DirectoryManager {
 		return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool";
 	}
 	
-	public String getCacheBinaryPathFor(Job job) {
-		return configuration.getWoolCacheDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip";
-	}
-	
-	public String getSharedBinaryPathFor(Job job) {
-		return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip";
-	}
-	
 	public boolean isSharedEnabled() {
 		return configuration.getSharedDownloadsDirectory() != null && configuration.getSharedDownloadsDirectory().exists();
 	}
 	
-	public boolean copyBinaryFromSharedToCache(Job job) {
-		return copyFileFromSharedToCache(getSharedBinaryPathFor(job), getCacheBinaryPathFor(job));
-	}
-	
 	public boolean copyChunkFromSharedToCache(Chunk chunk) {
 		return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk));
 	}

src/main/java/com/sheepit/client/Job.java

@@ -65,6 +65,7 @@ import java.util.regex.Pattern;
 	private String frameNumber;
 	private List<Chunk> archiveChunks;
 	private String rendererMD5;
+	private List<Chunk> rendererChunks;
 	private String id;
 	private String outputImagePath;
 	
@@ -92,7 +93,7 @@ import java.util.regex.Pattern;
 	private Log log;
 	
 	public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
-			String script_, List<Chunk> archiveChunks_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_,
+			String script_, List<Chunk> archiveChunks_, String rendererMd5_, List<Chunk> rendererChunks_, String name_, char[] password_, boolean synchronous_upload_,
 			String update_method_) {
 		configuration = config_;
 		id = id_;
@@ -103,6 +104,7 @@ import java.util.regex.Pattern;
 		validationUrl = validationUrl_;
 		archiveChunks = archiveChunks_;
 		rendererMD5 = rendererMd5_;
+		rendererChunks = rendererChunks_;
 		name = name_;
 		password = password_.clone();
 		synchronousUpload = synchronous_upload_;

src/main/java/com/sheepit/client/Server.java

@@ -400,7 +400,7 @@ public class Server extends Thread {
 					jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator),
 					jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl,
 					jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(),
-					jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
+					jobData.getRenderTask().getRendererInfos().getChunks(), jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
 					"1".equals(jobData.getRenderTask().getSynchronousUpload()), jobData.getRenderTask().getRendererInfos().getUpdateMethod());
 		}
 		catch (SheepItException e) {
@@ -688,7 +688,7 @@ public class Server extends Thread {
 			try {
 				String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase();
 				String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length());
-				if (".zip".equals(extension) || ".wool".equals(extension)) {
+				if (".wool".equals(extension)) {
 					// node_file.setAttribute("md5", name);
 					FileMD5 fileMD5 = new FileMD5();
 					fileMD5.setMd5(name);
@@ -728,7 +728,6 @@ public class Server extends Thread {
 					
 					for(String path: paths) {
 						new File(path + ".wool").delete();
-						new File(path + ".zip").delete();
 						
 						Utils.delete(new File(path));
 					}

src/main/java/com/sheepit/client/Utils.java

@@ -35,6 +35,7 @@ import java.io.StringWriter;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
@@ -393,4 +394,12 @@ public class Utils {
 		}
 		return true;
 	}
+	
+	public static <T> List<T> concatWithCollection(List<T> list1, List<T> list2) {
+		List<T> resultList = new ArrayList<>(list1.size() + list1.size());
+		resultList.addAll(list1);
+		resultList.addAll(list2);
+		
+		return resultList;
+	}
 }

src/main/java/com/sheepit/client/datamodel/RendererInfos.java

@@ -4,8 +4,11 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.ToString;
 import org.simpleframework.xml.Attribute;
+import org.simpleframework.xml.ElementList;
 import org.simpleframework.xml.Root;
 
+import java.util.List;
+
 @NoArgsConstructor @Root(strict = false, name = "renderer") @ToString public class RendererInfos {
 	
 	@Attribute(name = "md5") @Getter private String md5;
@@ -14,4 +17,5 @@ import org.simpleframework.xml.Root;
 	
 	@Attribute(name = "update_method") @Getter private String updateMethod;
 	
+	@ElementList(name = "chunks") @Getter private List<Chunk> chunks;
 }