Merge branch 'fix/download-progress' into 'master'

Fix: download progress

See merge request sheepitrenderfarm/client!342

src/main/java/com/sheepit/client/Client.java

@@ -796,15 +796,15 @@ import okhttp3.HttpUrl;
 		return err;
 	}
 	
-	private Error.Type downloadChunks(List<Chunk> chunks, String status) throws SheepItException {
+	private Error.Type downloadChunks(DownloadItem downloadInfos, String status) throws SheepItException {
-		int threads = Math.max(1, Math.min(chunks.size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
+		int threads = Math.max(1, Math.min(downloadInfos.getChunks().size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
 		
 		ExecutorService executor = Executors.newFixedThreadPool(threads);
 		List<Callable<Error.Type>> tasks = new ArrayList<>();
 		
-		this.gui.getDownloadProgress().reset(status);
+		this.gui.getDownloadProgress().reset(status, downloadInfos.getTotalSize());
 		
-		for (Chunk chunk : chunks) {
+		for (Chunk chunk : downloadInfos.getChunks()) {
 			Callable<Type> downloadTask = () -> {
 				DownloadManager downloadManager  = new DownloadManager(
 					this.server,
@@ -842,11 +842,11 @@ import okhttp3.HttpUrl;
 	}
 	
 	protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
-		return this.downloadChunks(ajob.getArchiveChunks(), "Downloading Project");
+		return this.downloadChunks(ajob.getProjectDownload(), "Downloading Project");
 	}
 	
 	protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
-		return this.downloadChunks(ajob.getRendererChunks(), "Downloading Blender");
+		return this.downloadChunks(ajob.getRendererDownload(), "Downloading Blender");
 	}
 	
 	protected void removeSceneDirectory(Job ajob) {
@@ -860,7 +860,7 @@ import okhttp3.HttpUrl;
 		File rendererPathFile = new File(rendererPath);
 		
 		// chunk files are already downloaded, either on shared directory or cache directory
-		for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererChunks(), ajob.getArchiveChunks())) {
+		for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererDownload().getChunks(), ajob.getProjectDownload().getChunks())) {
 			if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
 				this.gui.status("Copying chunk from common directory");
 				if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
@@ -878,7 +878,7 @@ import okhttp3.HttpUrl;
 			this.log.debug("Client::prepareWorkingDirectory Extracting renderer into " + rendererPath);
 			
 			// unzip the archive
-			ret = Utils.unzipChunksIntoDirectory(ajob.getRendererChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
+			ret = Utils.unzipChunksIntoDirectory(ajob.getRendererDownload().getChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
 			if (ret != 0) {
 				this.log.error("Client::prepareWorkingDirectory, error(1) with Utils.unzipChunksIntoDirectory(" + rendererPath + ") returned " + ret);
 				this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
@@ -906,14 +906,14 @@ import okhttp3.HttpUrl;
 			
 			Instant startUnzip = Instant.now();
 			ret = Utils.unzipChunksIntoDirectory(
-				ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
+				ajob.getProjectDownload().getChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
 				scenePath,
 				ajob.getPassword(),
 				log);
 			
 			Instant stopUnzip = Instant.now();
 			Duration unzipDuration = Duration.between(startUnzip, stopUnzip);
-			log.debug("Unzipping " + ajob.getArchiveChunks().size() + " chunks of \"" + ajob.getName() + "\" took " + unzipDuration.toSeconds() + "s");
+			log.debug("Unzipping " + ajob.getProjectDownload().getChunks().size() + " chunks of \"" + ajob.getName() + "\" took " + unzipDuration.toSeconds() + "s");
 			
 			if (ret != 0) {
 				this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);

src/main/java/com/sheepit/client/DownloadItem.java

@@ -0,0 +1,17 @@
+package com.sheepit.client;
+
+import com.sheepit.client.datamodel.Chunk;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * Download item is a zip file split in partial data (chunk)
+ */
+@Data @AllArgsConstructor public class DownloadItem {
+	private List<Chunk> chunks;
+	private long totalSize;
+	private String MD5;
+	
+}

src/main/java/com/sheepit/client/DownloadProgress.java

@@ -31,16 +31,12 @@ public class DownloadProgress {
 		this.gui = gui;
 	}
 	
-	public synchronized void reset(String pattern) {
+	public synchronized void reset(String pattern, long total) {
 		this.guiPattern = pattern;
-		this.total = 0;
+		this.total = total;
 		this.partial = 0;
 	}
 	
-	public synchronized void addTotal(long size) {
-		this.total += size;
-	}
-	
 	public synchronized void addProgress(long progress) {
 		this.partial += progress;
 		if (this.total != 0) {

src/main/java/com/sheepit/client/Job.java

@@ -21,7 +21,6 @@ package com.sheepit.client;
 
 import com.sheepit.client.Configuration.ComputeType;
 import com.sheepit.client.Error.Type;
-import com.sheepit.client.datamodel.Chunk;
 import com.sheepit.client.os.OS;
 import lombok.Data;
 import lombok.Getter;
@@ -63,9 +62,8 @@ import java.util.regex.Pattern;
 	public static final int SHOW_BASE_ICON = -1;
 	
 	private String frameNumber;
-	private List<Chunk> archiveChunks;
+	private DownloadItem projectDownload;
-	private String rendererMD5;
+	private DownloadItem rendererDownload;
-	private List<Chunk> rendererChunks;
 	private String id;
 	private String outputImagePath;
 	
@@ -93,7 +91,7 @@ import java.util.regex.Pattern;
 	private Log log;
 	
 	public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
-			String script_, List<Chunk> archiveChunks_, String rendererMd5_, List<Chunk> rendererChunks_, String name_, char[] password_, boolean synchronous_upload_,
+			String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
 			String update_method_) {
 		configuration = config_;
 		id = id_;
@@ -102,9 +100,8 @@ import java.util.regex.Pattern;
 		useGPU = use_gpu;
 		rendererCommand = command_;
 		validationUrl = validationUrl_;
-		archiveChunks = archiveChunks_;
+		projectDownload = projectDownload_;
-		rendererMD5 = rendererMd5_;
+		rendererDownload = rendererDownload_;
-		rendererChunks = rendererChunks_;
 		name = name_;
 		password = password_.clone();
 		synchronousUpload = synchronous_upload_;
@@ -150,7 +147,7 @@ import java.util.regex.Pattern;
 	public String toString() {
 		return String
 				.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
-						frameNumber, archiveChunks, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
+						frameNumber, projectDownload.getMD5(), rendererDownload.getMD5(), id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
 	}
 	
 	public String getPrefixOutputImage() {
@@ -158,7 +155,7 @@ import java.util.regex.Pattern;
 	}
 	
 	public String getRendererDirectory() {
-		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererMD5;
+		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererDownload.getMD5();
 	}
 	
 	public String getRendererPath() {

src/main/java/com/sheepit/client/Server.java

@@ -395,12 +395,22 @@ public class Server extends Thread {
 			}
 
 			String validationUrl = URLDecoder.decode(jobData.getRenderTask().getValidationUrl(), "UTF-8");
-
+			
+			DownloadItem projectDownload = new DownloadItem(
+				jobData.getRenderTask().getChunks(),
+				jobData.getRenderTask().getSize(),
+				jobData.getRenderTask().getMd5());
+			
+			DownloadItem rendererDownload = new DownloadItem(
+				jobData.getRenderTask().getRendererInfos().getChunks(),
+				jobData.getRenderTask().getRendererInfos().getSize(),
+				jobData.getRenderTask().getRendererInfos().getMd5());
+			
 			return new Job(this.user_config, this.client.getGui(), this.client.getLog(), jobData.getRenderTask().getId(),
 					jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator),
 					jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl,
-					jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(),
+					jobData.getRenderTask().getScript(), projectDownload, rendererDownload,
-					jobData.getRenderTask().getRendererInfos().getChunks(), jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
+					jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
 					"1".equals(jobData.getRenderTask().getSynchronousUpload()), jobData.getRenderTask().getRendererInfos().getUpdateMethod());
 		}
 		catch (SheepItException e) {
@@ -508,10 +518,6 @@ public class Server extends Thread {
 			int len;
 			long written = 0;
 			
-			if (size != -1) {
-				gui_.getDownloadProgress().addTotal(size);
-			}
-			
 			this.log.debug("Downloading file from " + response.request().url().host());
 			LocalDateTime startRequestTime = LocalDateTime.now();
 			

src/main/java/com/sheepit/client/datamodel/RenderTask.java

@@ -16,6 +16,10 @@ import java.util.List;
 	
 	@Attribute(name = "use_gpu") private int useGpu;
 	
+	@Attribute(name = "size") private int size;
+	
+	@Attribute(name = "md5") @Getter private String md5;
+	
 	@ElementList(name = "chunks") private List<Chunk> chunks;
 	
 	@Attribute(name = "path") private String path;

src/main/java/com/sheepit/client/datamodel/RendererInfos.java

@@ -18,4 +18,6 @@ import java.util.List;
 	@Attribute(name = "update_method") @Getter private String updateMethod;
 	
 	@ElementList(name = "chunks") @Getter private List<Chunk> chunks;
+	
+	@Attribute(name = "size") @Getter private int size;
 }