Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/split-project-in-chunk' into 'master'

Browse Source 
Feat: split project zip in chunks

See merge request sheepitrenderfarm/client!239
This commit is contained in:
Sheepit Renderfarm
2023-09-19 17:14:49 +00:00
parent 987cf3f0aa e803da9a3d
commit b8e5b805af
7 changed files with 139 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/com/sheepit/client/Client.java
View File

@@ -44,9 +44,11 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.sheepit.client.Error.ServerCode;
import com.sheepit.client.Error.Type;
import com.sheepit.client.datamodel.Chunk;
import com.sheepit.client.exception.SheepItException;
import com.sheepit.client.exception.SheepItExceptionBadResponseFromServer;
import com.sheepit.client.exception.SheepItExceptionNoRendererAvailable;
@@ -859,13 +861,18 @@ import okhttp3.HttpUrl;
}
protected Error.Type downloadSceneFile(Job ajob_) throws SheepItException {
return this.downloadFile(ajob_, ajob_.getRequiredSceneArchivePath(), ajob_.getSceneMD5(),
String.format(LOCALE, "%s?type=job&job=%s", this.server.getPage("download-archive"), ajob_.getId()), "project");
for (Chunk chunk : ajob_.getArchiveChunks()) {
Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), "project");
if (ret != Type.OK) {
return ret;
}
}
return Type.OK;
}
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
String.format(LOCALE, "%s?type=binary&job=%s", this.server.getPage("download-archive"), ajob.getId()), "renderer");
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer");
}
private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException {
@@ -900,7 +907,7 @@ import okhttp3.HttpUrl;
}
}
else {
// The file doesn't yet existing not is being downloaded by another client, so immediately create the file with zero bytes to allow early
// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
// detection by other concurrent clients and start downloading process
try {
File file = new File(local_path + ".partial");
@@ -1008,7 +1015,7 @@ import okhttp3.HttpUrl;
if (!new File(renderer_archive).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
copySharedArchive(bestRendererArchive, renderer_archive);
copySharedChunk(bestRendererArchive, renderer_archive);
}
if (!renderer_path_file.exists()) {
@@ -1035,15 +1042,17 @@ import okhttp3.HttpUrl;
}
}
String bestSceneArchive = ajob.getRequiredSceneArchivePath();
String scene_archive = ajob.getSceneArchivePath();
String scene_path = ajob.getSceneDirectory();
File scene_path_file = new File(scene_path);
if (!new File(scene_archive).exists()) {
this.gui.status("Copying scene from common directory");
copySharedArchive(bestSceneArchive, scene_archive);
for (Chunk chunk: ajob.getArchiveChunks()) {
if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) {
this.gui.status("Copying chunk from common directory");
copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId()));
}
}
/// download the chunks
if (!scene_path_file.exists()) {
// we create the directory
@@ -1052,11 +1061,15 @@ import okhttp3.HttpUrl;
this.gui.status("Extracting project");
// unzip the archive
ret = Utils.unzipFileIntoDirectory(scene_archive, scene_path, ajob.getPassword(), log);
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()),
scene_path,
ajob.getPassword(),
log);
if (ret != 0) {
this.log.error(
"Client::prepareWorkingDirectory, error(2) with Utils.unzipFileIntoDirectory(" + scene_archive + ", " + scene_path + ") returned "
+ ret);
this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);
this.gui.error(String.format("Unable to extract the scene (error %d)", ret));
return -2;
}
@@ -1065,7 +1078,7 @@ import okhttp3.HttpUrl;
return 0;
}
private void copySharedArchive(String existingArchive, String targetArchive) {
private void copySharedChunk(String existingArchive, String targetArchive) {
Path existingArchivePath = Paths.get(existingArchive);
Path targetArchivePath = Paths.get(targetArchive);
try {

2
src/main/java/com/sheepit/client/Configuration.java
View File

@@ -258,7 +258,7 @@ import lombok.Data;
try {
String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
if (extension.equals(".zip")) {
if (extension.equals(".zip") || extension.equals(".wool")) {
// check if the md5 of the file is ok
String md5_local = Utils.md5(file.getAbsolutePath());

23
src/main/java/com/sheepit/client/Job.java
View File

@@ -21,6 +21,7 @@ package com.sheepit.client;
import com.sheepit.client.Configuration.ComputeType;
import com.sheepit.client.Error.Type;
import com.sheepit.client.datamodel.Chunk;
import com.sheepit.client.os.OS;
import lombok.Data;
import lombok.Getter;
@@ -62,7 +63,7 @@ import java.util.regex.Pattern;
public static final int SHOW_BASE_ICON = -1;
private String frameNumber;
private String sceneMD5;
private List<Chunk> archiveChunks;
private String rendererMD5;
private String id;
private String outputImagePath;
@@ -90,7 +91,7 @@ import java.util.regex.Pattern;
private Log log;
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
String script_, String sceneMd5_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_,
String script_, List<Chunk> archiveChunks_, String rendererMd5_, String name_, char[] password_, boolean synchronous_upload_,
String update_method_) {
configuration = config_;
id = id_;
@@ -99,7 +100,7 @@ import java.util.regex.Pattern;
useGPU = use_gpu;
rendererCommand = command_;
validationUrl = validationUrl_;
sceneMD5 = sceneMd5_;
archiveChunks = archiveChunks_;
rendererMD5 = rendererMd5_;
name = name_;
password = password_;
@@ -134,8 +135,8 @@ import java.util.regex.Pattern;
public String toString() {
return String
.format("Job (numFrame '%s' sceneMD5 '%s' rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
frameNumber, sceneMD5, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
frameNumber, archiveChunks, rendererMD5, id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
}
public String getPrefixOutputImage() {
@@ -163,25 +164,25 @@ import java.util.regex.Pattern;
return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
}
public String getRequiredSceneArchivePath() {
public String getRequiredProjectChunkPath(String chunk) {
if (configuration.getSharedDownloadsDirectory() != null) {
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip";
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
}
else {
return getSceneArchivePath();
return getSceneArchiveChunkPath(chunk);
}
}
public String getSceneDirectory() {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5;
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id;
}
public String getScenePath() {
return getSceneDirectory() + File.separator + this.path;
}
public String getSceneArchivePath() {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + sceneMD5 + ".zip";
public String getSceneArchiveChunkPath(String chunk) {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
}
public Error.Type render(Observer renderStarted) {

31
src/main/java/com/sheepit/client/Server.java
View File

@@ -401,7 +401,7 @@ public class Server extends Thread {
return new Job(this.user_config, this.client.getGui(), this.client.getLog(), jobData.getRenderTask().getId(),
jobData.getRenderTask().getFrame(), jobData.getRenderTask().getPath().replace("/", File.separator),
jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getRendererInfos().getCommandline(), validationUrl,
jobData.getRenderTask().getScript(), jobData.getRenderTask().getArchive_md5(), jobData.getRenderTask().getRendererInfos().getMd5(),
jobData.getRenderTask().getScript(), jobData.getRenderTask().getChunks(), jobData.getRenderTask().getRendererInfos().getMd5(),
jobData.getRenderTask().getName(), jobData.getRenderTask().getPassword(),
jobData.getRenderTask().getSynchronous_upload().equals("1"), jobData.getRenderTask().getRendererInfos().getUpdate_method());
}
@@ -491,6 +491,7 @@ public class Server extends Thread {
}
public Error.Type HTTPGetFile(String url_, String destination_, Gui gui_, String status_) throws SheepItException {
this.log.debug("Server::HTTPGetFile destination: " + destination_);
InputStream is = null;
OutputStream output = null;
@@ -694,7 +695,7 @@ public class Server extends Thread {
try {
String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase();
String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length());
if (extension.equals(".zip")) {
if (extension.equals(".zip") || extension.equals(".wool")) {
// node_file.setAttribute("md5", name);
FileMD5 fileMD5 = new FileMD5();
fileMD5.setMd5(name);
@@ -724,24 +725,20 @@ public class Server extends Thread {
if (fileMD5s != null && fileMD5s.isEmpty() == false) {
for (FileMD5 fileMD5 : fileMD5s) {
if ("delete".equals(fileMD5.getAction()) && fileMD5.getMd5() != null && fileMD5.getMd5().isEmpty() == false) {
String path = this.user_config.getWorkingDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5();
this.log.debug("Server::handleFileMD5DeleteDocument delete old file " + path);
File file_to_delete = new File(path + ".zip");
file_to_delete.delete();
Utils.delete(new File(path));
//also delete in binary cache
path = this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5();
file_to_delete = new File(path + ".zip");
file_to_delete.delete();
Utils.delete(new File(path));
List<String> paths = new ArrayList<>();
paths.add(this.user_config.getStorageDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5()); //also delete in binary cache
paths.add(this.user_config.getWorkingDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5());
// If we are using a shared downloads directory, then delete the file from the shared downloads directory as well :)
if (this.user_config.getSharedDownloadsDirectory() != null) {
String commonCacheFile = this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separatorChar + fileMD5.getMd5();
this.log.debug("Server::handleFileMD5DeleteDocument delete common file " + commonCacheFile + ".zip");
file_to_delete = new File(commonCacheFile + ".zip");
file_to_delete.delete();
paths.add(this.user_config.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + fileMD5.getMd5());
}
for(String path: paths) {
new File(path + ".wool").delete();
new File(path + ".zip").delete();
Utils.delete(new File(path));
}
}
}

67
src/main/java/com/sheepit/client/Utils.java
View File

@@ -22,13 +22,18 @@ package com.sheepit.client;
import com.sheepit.client.Error.ServerCode;
import net.lingala.zip4j.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import net.lingala.zip4j.io.inputstream.ZipInputStream;
import net.lingala.zip4j.model.LocalFileHeader;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -36,12 +41,10 @@ import java.io.StringWriter;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
@@ -77,6 +80,64 @@ public class Utils {
return 0;
}
public static int unzipChunksIntoDirectory(List<String> full_path_chunks, String destinationDirectory, char[] password, Log log) {
try {
// STEP 1: assemble the chunks into an actual zip (in RAM)
ByteArrayOutputStream unzippedData = new ByteArrayOutputStream();
for (String full_path_chunk: full_path_chunks) {
byte[] data = Files.readAllBytes(Paths.get(full_path_chunk));
unzippedData.write(data);
}
byte[] full_data = unzippedData.toByteArray();
// STEP 2: unzip the zip like before
ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(full_data));
if (password != null) {
zipInputStream.setPassword(password);
}
LocalFileHeader fileHeader = null;
while ((fileHeader = zipInputStream.getNextEntry()) != null) {
String outFilePath = destinationDirectory + File.separator + fileHeader.getFileName();
File outFile = new File(outFilePath);
//Checks if the file is a directory
if (fileHeader.isDirectory()) {
outFile.mkdirs();
continue;
}
File parentDir = outFile.getParentFile();
if (parentDir.exists() == false) {
parentDir.mkdirs();
}
FileOutputStream os = new FileOutputStream(outFile);
int readLen = -1;
byte[] buff = new byte[1024];
//Loop until End of File and write the contents to the output stream
while ((readLen = zipInputStream.read(buff)) != -1) {
os.write(buff, 0, readLen);
}
os.close();
}
zipInputStream.close();
}
catch (IOException e) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
log.debug("Utils::unzipChunksIntoDirectory exception " + e + " stacktrace: " + sw.toString());
return -1;
}
return 0;
}
public static String md5(String path_of_file_) {
Md5 md5 = new Md5();
return md5.get(path_of_file_);

16
src/main/java/com/sheepit/client/datamodel/Chunk.java Normal file
View File

@@ -0,0 +1,16 @@
package com.sheepit.client.datamodel;
import lombok.Data;
import lombok.ToString;
import org.simpleframework.xml.Attribute;
import org.simpleframework.xml.Root;
@Root(strict = false, name = "chunk") @Data @ToString public class Chunk {
@Attribute private String md5;
@Attribute private String id;
public Chunk() {
}
}

5
src/main/java/com/sheepit/client/datamodel/RenderTask.java
View File

@@ -4,15 +4,18 @@ import lombok.Getter;
import lombok.ToString;
import org.simpleframework.xml.Attribute;
import org.simpleframework.xml.Element;
import org.simpleframework.xml.ElementList;
import org.simpleframework.xml.Root;
import java.util.List;
@Root(strict = false, name = "job") @Getter @ToString public class RenderTask {
@Attribute(name = "id") private String id;
@Attribute(name = "use_gpu") private int useGpu;
@Attribute(name = "archive_md5") private String archive_md5;
@ElementList(name = "chunks") private List<Chunk> chunks;
@Attribute(name = "path") private String path;