Merge branch 'feat/split-project-in-chunk' into 'master'

Feat: split project zip in chunks

See merge request sheepitrenderfarm/client!239

src/main/java/com/sheepit/client/Client.java

@@ -44,9 +44,11 @@ import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import com.sheepit.client.Error.ServerCode;
 import com.sheepit.client.Error.Type;
+import com.sheepit.client.datamodel.Chunk;
 import com.sheepit.client.exception.SheepItException;
 import com.sheepit.client.exception.SheepItExceptionBadResponseFromServer;
 import com.sheepit.client.exception.SheepItExceptionNoRendererAvailable;
@@ -859,13 +861,18 @@ import okhttp3.HttpUrl;
 	}
 	
 	protected Error.Type downloadSceneFile(Job ajob_) throws SheepItException {
-		return this.downloadFile(ajob_, ajob_.getRequiredSceneArchivePath(), ajob_.getSceneMD5(),
+		for (Chunk chunk : ajob_.getArchiveChunks()) {
-				String.format(LOCALE, "%s?type=job&job=%s", this.server.getPage("download-archive"), ajob_.getId()), "project");
+			Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), "project");
+			if (ret != Type.OK) {
+				return ret;
+			}
+		}
+		return Type.OK;
 	}
 	
 	protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
 		return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
-				String.format(LOCALE, "%s?type=binary&job=%s", this.server.getPage("download-archive"), ajob.getId()), "renderer");
+				String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer");
 	}
 	
 	private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException {
@@ -900,7 +907,7 @@ import okhttp3.HttpUrl;
 					}
 				}
 				else {
-					// The file doesn't yet existing not is being downloaded by another client, so immediately create the file with zero bytes to allow early
+					// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
 					// detection by other concurrent clients and start downloading process
 					try {
 						File file = new File(local_path + ".partial");
@@ -1008,7 +1015,7 @@ import okhttp3.HttpUrl;
 		if (!new File(renderer_archive).exists()) {
 			this.gui.status("Copying renderer from shared downloads directory");
 			
-			copySharedArchive(bestRendererArchive, renderer_archive);
+			copySharedChunk(bestRendererArchive, renderer_archive);
 		}
 		
 		if (!renderer_path_file.exists()) {
@@ -1035,15 +1042,17 @@ import okhttp3.HttpUrl;
 			}
 		}
 		
-		String bestSceneArchive = ajob.getRequiredSceneArchivePath();
-		String scene_archive = ajob.getSceneArchivePath();
 		String scene_path = ajob.getSceneDirectory();
 		File scene_path_file = new File(scene_path);
 		
-		if (!new File(scene_archive).exists()) {
+		for (Chunk chunk: ajob.getArchiveChunks()) {
-			this.gui.status("Copying scene from common directory");
+			if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) {
-			copySharedArchive(bestSceneArchive, scene_archive);
+				this.gui.status("Copying chunk from common directory");
+				copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId()));
 			}
+		}
+		
+		/// download the chunks
 		
 		if (!scene_path_file.exists()) {
 			// we create the directory
@@ -1052,11 +1061,15 @@ import okhttp3.HttpUrl;
 			this.gui.status("Extracting project");
 			
 			// unzip the archive
-			ret = Utils.unzipFileIntoDirectory(scene_archive, scene_path, ajob.getPassword(), log);
+			
+			
+			ret = Utils.unzipChunksIntoDirectory(
+				ajob.getArchiveChunks().stream().map(input ->  ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()),
+				scene_path,
+				ajob.getPassword(),
+				log);
 			if (ret != 0) {
-				this.log.error(
+				this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);
-						"Client::prepareWorkingDirectory, error(2) with Utils.unzipFileIntoDirectory(" + scene_archive + ", " + scene_path + ") returned "
-								+ ret);
 				this.gui.error(String.format("Unable to extract the scene (error %d)", ret));
 				return -2;
 			}
@@ -1065,7 +1078,7 @@ import okhttp3.HttpUrl;
 		return 0;
 	}
 
-	private void copySharedArchive(String existingArchive, String targetArchive) {
+	private void copySharedChunk(String existingArchive, String targetArchive) {
 		Path existingArchivePath = Paths.get(existingArchive);
 		Path targetArchivePath = Paths.get(targetArchive);
 		try {

src/main/java/com/sheepit/client/Configuration.java

@@ -258,7 +258,7 @@ import lombok.Data;
 					try {
 						String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
 						String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
-						if (extension.equals(".zip")) {
+						if (extension.equals(".zip") || extension.equals(".wool")) {
 							// check if the md5 of the file is ok
 							String md5_local = Utils.md5(file.getAbsolutePath());
 							

src/main/java/com/sheepit/client/Job.java

@@ -21,6 +21,7 @@ package com.sheepit.client;
 
 import com.sheepit.client.Configuration.ComputeType;
 import com.sheepit.client.Error.Type;
+import com.sheepit.client.datamodel.Chunk;
 import com.sheepit.client.os.OS;
 import lombok.Data;
 import lombok.Getter;
@@ -62,7 +63,7 @@ import java.util.regex.Pattern;
 	public static final int SHOW_BASE_ICON = -1;
 	
 	private String frameNumber;
-	private String sceneMD5;
+	private List<Chunk> archiveChunks;
 	private String rendererMD5;
 	private String id;
 	private String outputImagePath;
@@ -90,7 +91,7 @@ import java.util.regex.Pattern;
 	private Log log;
 	
 	public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
-			String script_, String sceneMd5_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_,
+			String script_, List<Chunk> archiveChunks_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_,
 			String update_method_) {
 		configuration = config_;
 		id = id_;
@@ -99,7 +100,7 @@ import java.util.regex.Pattern;
 		useGPU = use_gpu;
 		rendererCommand = command_;
 		validationUrl = validationUrl_;
-		sceneMD5 = sceneMd5_;
+		archiveChunks = archiveChunks_;
 		rendererMD5 = rendererMd5_;
 		name = name_;
 		password = password_;
@@ -134,8 +135,8 @@ import java.util.regex.Pattern;
 	
 	public String toString() {
 		return String
-				.format("Job (numFrame '%s' sceneMD5 '%s' rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
+				.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
-						frameNumber, sceneMD5, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
+						frameNumber, archiveChunks, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
 	}
 	
 	public String getPrefixOutputImage() {
@@ -163,25 +164,25 @@ import java.util.regex.Pattern;
 		return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
 	}
 	
-	public String getRequiredSceneArchivePath() {
+	public String getRequiredProjectChunkPath(String chunk) {
 		if (configuration.getSharedDownloadsDirectory() != null) {
-			return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip";
+			return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
 		}
 		else {
-			return getSceneArchivePath();
+			return getSceneArchiveChunkPath(chunk);
 		}
 	}
 	
 	public String getSceneDirectory() {
-		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5;
+		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id;
 	}
 	
 	public String getScenePath() {
 		return getSceneDirectory() + File.separator + this.path;
 	}
 	
-	public String getSceneArchivePath() {
+	public String getSceneArchiveChunkPath(String chunk) {
-		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip";
+		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
 	}
 	
 	public Error.Type render(Observer renderStarted) {

src/main/java/com/sheepit/client/Server.java

@@ -401,7 +401,7 @@ public class Server extends Thread {
 			return new Job(this.user_config, this.client.getGui(), this.client.getLog(), jobData.getRenderTask().getId(),
 					jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator),
 					jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl,
-					jobData.getRenderTask().getScript(), jobData.getRenderTask().getArchive_md5(), jobData.getRenderTask().getRendererInfos().getMd5(),
+					jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(),
 					jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
 					jobData.getRenderTask().getSynchronous_upload().equals("1"), jobData.getRenderTask().getRendererInfos().getUpdate_method());
 		}
@@ -491,6 +491,7 @@ public class Server extends Thread {
 	}
 	
 	public Error.Type HTTPGetFile(String url_, String destination_, Gui gui_, String status_) throws SheepItException {
+		this.log.debug("Server::HTTPGetFile destination: " + destination_);
 		InputStream is = null;
 		OutputStream output = null;
 
@@ -694,7 +695,7 @@ public class Server extends Thread {
 			try {
 				String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase();
 				String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length());
-				if (extension.equals(".zip")) {
+				if (extension.equals(".zip") || extension.equals(".wool")) {
 					// node_file.setAttribute("md5", name);
 					FileMD5 fileMD5 = new FileMD5();
 					fileMD5.setMd5(name);
@@ -724,24 +725,20 @@ public class Server extends Thread {
 		if (fileMD5s != null && fileMD5s.isEmpty() == false) {
 			for (FileMD5 fileMD5 : fileMD5s) {
 				if ("delete".equals(fileMD5.getAction()) && fileMD5.getMd5() != null && fileMD5.getMd5().isEmpty() == false) {
-					String path = this.user_config.getWorkingDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5();
+					List<String> paths = new ArrayList<>();
-					this.log.debug("Server::handleFileMD5DeleteDocument delete old file " + path);
+					paths.add(this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5()); //also delete in binary cache
-					File file_to_delete = new File(path + ".zip");
+					paths.add(this.user_config.getWorkingDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5());
-					file_to_delete.delete();
-					Utils.delete(new File(path));
-					
-					//also delete in binary cache
-					path = this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5();
-					file_to_delete = new File(path + ".zip");
-					file_to_delete.delete();
-					Utils.delete(new File(path));
 					
 					// If we are using a shared downloads directory, then delete the file from the shared downloads directory as well :)
 					if (this.user_config.getSharedDownloadsDirectory() != null) {
-						String commonCacheFile = this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5();
+						paths.add(this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5());
-						this.log.debug("Server::handleFileMD5DeleteDocument delete common file " + commonCacheFile + ".zip");
+					}
-						file_to_delete = new File(commonCacheFile + ".zip");
+					
-						file_to_delete.delete();
+					for(String path: paths) {
+						new File(path + ".wool").delete();
+						new File(path + ".zip").delete();
+						
+						Utils.delete(new File(path));
 					}
 				}
 			}

src/main/java/com/sheepit/client/Utils.java

@@ -22,13 +22,18 @@ package com.sheepit.client;
 import com.sheepit.client.Error.ServerCode;
 import net.lingala.zip4j.ZipFile;
 import net.lingala.zip4j.exception.ZipException;
+import net.lingala.zip4j.io.inputstream.ZipInputStream;
+import net.lingala.zip4j.model.LocalFileHeader;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 
 import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
@@ -36,12 +41,10 @@ import java.io.StringWriter;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.security.DigestInputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
 import java.util.regex.Matcher;
@@ -77,6 +80,64 @@ public class Utils {
 		return 0;
 	}
 	
+	public static int unzipChunksIntoDirectory(List<String> full_path_chunks, String destinationDirectory, char[] password, Log log) {
+		try {
+			// STEP 1: assemble the chunks into an actual zip (in RAM)
+			ByteArrayOutputStream unzippedData = new ByteArrayOutputStream();
+			
+			for (String full_path_chunk: full_path_chunks) {
+				byte[] data = Files.readAllBytes(Paths.get(full_path_chunk));
+				
+				unzippedData.write(data);
+			}
+			byte[] full_data = unzippedData.toByteArray();
+			
+			// STEP 2: unzip the zip like before
+			ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(full_data));
+			if (password != null) {
+				zipInputStream.setPassword(password);
+			}
+			LocalFileHeader fileHeader = null;
+			while ((fileHeader = zipInputStream.getNextEntry()) != null) {
+				String outFilePath = destinationDirectory + File.separator + fileHeader.getFileName();
+				File outFile = new File(outFilePath);
+				
+				//Checks if the file is a directory
+				if (fileHeader.isDirectory()) {
+					outFile.mkdirs();
+					continue;
+				}
+				
+				File parentDir = outFile.getParentFile();
+				if (parentDir.exists() == false) {
+					parentDir.mkdirs();
+				}
+				
+				FileOutputStream os = new FileOutputStream(outFile);
+				
+				int readLen = -1;
+				byte[] buff = new byte[1024];
+				
+				//Loop until End of File and write the contents to the output stream
+				while ((readLen = zipInputStream.read(buff)) != -1) {
+					os.write(buff, 0, readLen);
+				}
+				
+				os.close();
+			}
+			zipInputStream.close();
+		}
+		catch (IOException e) {
+			StringWriter sw = new StringWriter();
+			PrintWriter pw = new PrintWriter(sw);
+			e.printStackTrace(pw);
+			log.debug("Utils::unzipChunksIntoDirectory exception " + e + " stacktrace: " + sw.toString());
+			return -1;
+		}
+		
+		return 0;
+	}
+	
 	public static String md5(String path_of_file_) {
 		Md5 md5 = new Md5();
 		return md5.get(path_of_file_);

src/main/java/com/sheepit/client/datamodel/Chunk.java

@@ -0,0 +1,16 @@
+package com.sheepit.client.datamodel;
+
+import lombok.Data;
+import lombok.ToString;
+import org.simpleframework.xml.Attribute;
+import org.simpleframework.xml.Root;
+
+@Root(strict = false, name = "chunk") @Data @ToString public class Chunk {
+	
+	@Attribute private String md5;
+	
+	@Attribute private String id;
+	
+	public Chunk() {
+	}
+}

src/main/java/com/sheepit/client/datamodel/RenderTask.java

@@ -4,15 +4,18 @@ import lombok.Getter;
 import lombok.ToString;
 import org.simpleframework.xml.Attribute;
 import org.simpleframework.xml.Element;
+import org.simpleframework.xml.ElementList;
 import org.simpleframework.xml.Root;
 
+import java.util.List;
+
 @Root(strict = false, name = "job") @Getter @ToString public class RenderTask {
 	
 	@Attribute(name = "id") private String id;
 	
 	@Attribute(name = "use_gpu")  private int useGpu;
 	
-	@Attribute(name = "archive_md5") private String archive_md5;
+	@ElementList(name = "chunks") private List<Chunk> chunks;
 	
 	@Attribute(name = "path") private String path;