Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'fix/download-progress' into 'master'

Browse Source 
Fix: download progress

See merge request sheepitrenderfarm/client!342
This commit is contained in:
Sheepit Renderfarm
2024-11-20 16:57:43 +00:00
parent 8d7a29cf7f 3d4c9a0487
commit ce6aa7ecd4
7 changed files with 55 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/sheepit/client/Client.java
View File

@@ -796,15 +796,15 @@ import okhttp3.HttpUrl;
return err;
}
private Error.Type downloadChunks(List<Chunk> chunks, String status) throws SheepItException {
int threads = Math.max(1, Math.min(chunks.size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
private Error.Type downloadChunks(DownloadItem downloadInfos, String status) throws SheepItException {
int threads = Math.max(1, Math.min(downloadInfos.getChunks().size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
ExecutorService executor = Executors.newFixedThreadPool(threads);
List<Callable<Error.Type>> tasks = new ArrayList<>();
this.gui.getDownloadProgress().reset(status);
this.gui.getDownloadProgress().reset(status, downloadInfos.getTotalSize());
for (Chunk chunk : chunks) {
for (Chunk chunk : downloadInfos.getChunks()) {
Callable<Type> downloadTask = () -> {
DownloadManager downloadManager = new DownloadManager(
this.server,
@@ -842,11 +842,11 @@ import okhttp3.HttpUrl;
}
protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
return this.downloadChunks(ajob.getArchiveChunks(), "Downloading Project");
return this.downloadChunks(ajob.getProjectDownload(), "Downloading Project");
}
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
return this.downloadChunks(ajob.getRendererChunks(), "Downloading Blender");
return this.downloadChunks(ajob.getRendererDownload(), "Downloading Blender");
}
protected void removeSceneDirectory(Job ajob) {
@@ -860,7 +860,7 @@ import okhttp3.HttpUrl;
File rendererPathFile = new File(rendererPath);
// chunk files are already downloaded, either on shared directory or cache directory
for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererChunks(), ajob.getArchiveChunks())) {
for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererDownload().getChunks(), ajob.getProjectDownload().getChunks())) {
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
this.gui.status("Copying chunk from common directory");
if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
@@ -878,7 +878,7 @@ import okhttp3.HttpUrl;
this.log.debug("Client::prepareWorkingDirectory Extracting renderer into " + rendererPath);
// unzip the archive
ret = Utils.unzipChunksIntoDirectory(ajob.getRendererChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
ret = Utils.unzipChunksIntoDirectory(ajob.getRendererDownload().getChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
if (ret != 0) {
this.log.error("Client::prepareWorkingDirectory, error(1) with Utils.unzipChunksIntoDirectory(" + rendererPath + ") returned " + ret);
this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
@@ -906,14 +906,14 @@ import okhttp3.HttpUrl;
Instant startUnzip = Instant.now();
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
ajob.getProjectDownload().getChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
scenePath,
ajob.getPassword(),
log);
Instant stopUnzip = Instant.now();
Duration unzipDuration = Duration.between(startUnzip, stopUnzip);
log.debug("Unzipping " + ajob.getArchiveChunks().size() + " chunks of \"" + ajob.getName() + "\" took " + unzipDuration.toSeconds() + "s");
log.debug("Unzipping " + ajob.getProjectDownload().getChunks().size() + " chunks of \"" + ajob.getName() + "\" took " + unzipDuration.toSeconds() + "s");
if (ret != 0) {
this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);

17
src/main/java/com/sheepit/client/DownloadItem.java Normal file
View File

@@ -0,0 +1,17 @@
package com.sheepit.client;
import com.sheepit.client.datamodel.Chunk;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
/**
* Download item is a zip file split in partial data (chunk)
*/
@Data @AllArgsConstructor public class DownloadItem {
private List<Chunk> chunks;
private long totalSize;
private String MD5;
}

8
src/main/java/com/sheepit/client/DownloadProgress.java
View File

@@ -31,16 +31,12 @@ public class DownloadProgress {
this.gui = gui;
}
public synchronized void reset(String pattern) {
public synchronized void reset(String pattern, long total) {
this.guiPattern = pattern;
this.total = 0;
this.total = total;
this.partial = 0;
}
public synchronized void addTotal(long size) {
this.total += size;
}
public synchronized void addProgress(long progress) {
this.partial += progress;
if (this.total != 0) {

17
src/main/java/com/sheepit/client/Job.java
View File

@@ -21,7 +21,6 @@ package com.sheepit.client;
import com.sheepit.client.Configuration.ComputeType;
import com.sheepit.client.Error.Type;
import com.sheepit.client.datamodel.Chunk;
import com.sheepit.client.os.OS;
import lombok.Data;
import lombok.Getter;
@@ -63,9 +62,8 @@ import java.util.regex.Pattern;
public static final int SHOW_BASE_ICON = -1;
private String frameNumber;
private List<Chunk> archiveChunks;
private String rendererMD5;
private List<Chunk> rendererChunks;
private DownloadItem projectDownload;
private DownloadItem rendererDownload;
private String id;
private String outputImagePath;
@@ -93,7 +91,7 @@ import java.util.regex.Pattern;
private Log log;
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
String script_, List<Chunk> archiveChunks_, String rendererMd5_, List<Chunk> rendererChunks_, String name_, char[] password_, boolean synchronous_upload_,
String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
String update_method_) {
configuration = config_;
id = id_;
@@ -102,9 +100,8 @@ import java.util.regex.Pattern;
useGPU = use_gpu;
rendererCommand = command_;
validationUrl = validationUrl_;
archiveChunks = archiveChunks_;
rendererMD5 = rendererMd5_;
rendererChunks = rendererChunks_;
projectDownload = projectDownload_;
rendererDownload = rendererDownload_;
name = name_;
password = password_.clone();
synchronousUpload = synchronous_upload_;
@@ -150,7 +147,7 @@ import java.util.regex.Pattern;
public String toString() {
return String
.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
frameNumber, archiveChunks, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
frameNumber, projectDownload.getMD5(), rendererDownload.getMD5(), id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
}
public String getPrefixOutputImage() {
@@ -158,7 +155,7 @@ import java.util.regex.Pattern;
}
public String getRendererDirectory() {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererMD5;
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererDownload.getMD5();
}
public String getRendererPath() {

18
src/main/java/com/sheepit/client/Server.java
View File

@@ -396,11 +396,21 @@ public class Server extends Thread {
String validationUrl = URLDecoder.decode(jobData.getRenderTask().getValidationUrl(), "UTF-8");
DownloadItem projectDownload = new DownloadItem(
jobData.getRenderTask().getChunks(),
jobData.getRenderTask().getSize(),
jobData.getRenderTask().getMd5());
DownloadItem rendererDownload = new DownloadItem(
jobData.getRenderTask().getRendererInfos().getChunks(),
jobData.getRenderTask().getRendererInfos().getSize(),
jobData.getRenderTask().getRendererInfos().getMd5());
return new Job(this.user_config, this.client.getGui(), this.client.getLog(), jobData.getRenderTask().getId(),
jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator),
jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl,
jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(),
jobData.getRenderTask().getRendererInfos().getChunks(), jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
jobData.getRenderTask().getScript(), projectDownload, rendererDownload,
jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
"1".equals(jobData.getRenderTask().getSynchronousUpload()), jobData.getRenderTask().getRendererInfos().getUpdateMethod());
}
catch (SheepItException e) {
@@ -508,10 +518,6 @@ public class Server extends Thread {
int len;
long written = 0;
if (size != -1) {
gui_.getDownloadProgress().addTotal(size);
}
this.log.debug("Downloading file from " + response.request().url().host());
LocalDateTime startRequestTime = LocalDateTime.now();

4
src/main/java/com/sheepit/client/datamodel/RenderTask.java
View File

@@ -16,6 +16,10 @@ import java.util.List;
@Attribute(name = "use_gpu") private int useGpu;
@Attribute(name = "size") private int size;
@Attribute(name = "md5") @Getter private String md5;
@ElementList(name = "chunks") private List<Chunk> chunks;
@Attribute(name = "path") private String path;

2
src/main/java/com/sheepit/client/datamodel/RendererInfos.java
View File

@@ -18,4 +18,6 @@ import java.util.List;
@Attribute(name = "update_method") @Getter private String updateMethod;
@ElementList(name = "chunks") @Getter private List<Chunk> chunks;
@Attribute(name = "size") @Getter private int size;
}