Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: split project zip in chunks

Browse Source
This commit is contained in:
Sheepit Renderfarm
2023-09-19 17:14:49 +00:00
parent 14c3e1ecc1
commit e803da9a3d
7 changed files with 139 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/com/sheepit/client/Client.java
View File

@@ -44,9 +44,11 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.sheepit.client.Error.ServerCode;
import com.sheepit.client.Error.Type;
import com.sheepit.client.datamodel.Chunk;
import com.sheepit.client.exception.SheepItException;
import com.sheepit.client.exception.SheepItExceptionBadResponseFromServer;
import com.sheepit.client.exception.SheepItExceptionNoRendererAvailable;
@@ -859,13 +861,18 @@ import okhttp3.HttpUrl;
}
protected Error.Type downloadSceneFile(Job ajob_) throws SheepItException {
return this.downloadFile(ajob_, ajob_.getRequiredSceneArchivePath(), ajob_.getSceneMD5(),
String.format(LOCALE, "%s?type=job&job=%s", this.server.getPage("download-archive"), ajob_.getId()), "project");
for (Chunk chunk : ajob_.getArchiveChunks()) {
Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), "project");
if (ret != Type.OK) {
return ret;
}
}
return Type.OK;
}
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
String.format(LOCALE, "%s?type=binary&job=%s", this.server.getPage("download-archive"), ajob.getId()), "renderer");
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer");
}
private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException {
@@ -900,7 +907,7 @@ import okhttp3.HttpUrl;
}
}
else {
// The file doesn't yet existing not is being downloaded by another client, so immediately create the file with zero bytes to allow early
// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
// detection by other concurrent clients and start downloading process
try {
File file = new File(local_path + ".partial");
@@ -1008,7 +1015,7 @@ import okhttp3.HttpUrl;
if (!new File(renderer_archive).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
copySharedArchive(bestRendererArchive, renderer_archive);
copySharedChunk(bestRendererArchive, renderer_archive);
}
if (!renderer_path_file.exists()) {
@@ -1035,16 +1042,18 @@ import okhttp3.HttpUrl;
}
}
String bestSceneArchive = ajob.getRequiredSceneArchivePath();
String scene_archive = ajob.getSceneArchivePath();
String scene_path = ajob.getSceneDirectory();
File scene_path_file = new File(scene_path);
if (!new File(scene_archive).exists()) {
this.gui.status("Copying scene from common directory");
copySharedArchive(bestSceneArchive, scene_archive);
for (Chunk chunk: ajob.getArchiveChunks()) {
if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) {
this.gui.status("Copying chunk from common directory");
copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId()));
}
}
/// download the chunks
if (!scene_path_file.exists()) {
// we create the directory
scene_path_file.mkdir();
@@ -1052,11 +1061,15 @@ import okhttp3.HttpUrl;
this.gui.status("Extracting project");
// unzip the archive
ret = Utils.unzipFileIntoDirectory(scene_archive, scene_path, ajob.getPassword(), log);
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()),
scene_path,
ajob.getPassword(),
log);
if (ret != 0) {
this.log.error(
"Client::prepareWorkingDirectory, error(2) with Utils.unzipFileIntoDirectory(" + scene_archive + ", " + scene_path + ") returned "
+ ret);
this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);
this.gui.error(String.format("Unable to extract the scene (error %d)", ret));
return -2;
}
@@ -1065,7 +1078,7 @@ import okhttp3.HttpUrl;
return 0;
}
private void copySharedArchive(String existingArchive, String targetArchive) {
private void copySharedChunk(String existingArchive, String targetArchive) {
Path existingArchivePath = Paths.get(existingArchive);
Path targetArchivePath = Paths.get(targetArchive);
try {