Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: use chunk for renderer

Browse Source
This commit is contained in:
Sheepit Renderfarm
2024-11-20 13:05:20 +00:00
parent f069f63f7a
commit 94b36241b0
7 changed files with 41 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/main/java/com/sheepit/client/Client.java
View File

@@ -796,18 +796,15 @@ import okhttp3.HttpUrl;
return err;
}
protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
int total = ajob.getArchiveChunks().size();
int threads = Math.max(1, Math.min(total, 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
private Error.Type downloadChunks(List<Chunk> chunks, String status) throws SheepItException {
int threads = Math.max(1, Math.min(chunks.size(), 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
ExecutorService executor = Executors.newFixedThreadPool(threads);
List<Callable<Error.Type>> tasks = new ArrayList<>();
this.gui.getDownloadProgress().reset("Downloading project");
this.gui.getDownloadProgress().reset(status);
for (int i = 0; i < total; i++) {
Chunk chunk = ajob.getArchiveChunks().get(i);
for (Chunk chunk : chunks) {
Callable<Type> downloadTask = () -> {
DownloadManager downloadManager = new DownloadManager(
this.server,
@@ -844,16 +841,12 @@ import okhttp3.HttpUrl;
return Type.OK;
}
protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
return this.downloadChunks(ajob.getArchiveChunks(), "Downloading Project");
}
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
this.gui.getDownloadProgress().reset("Downloading Blender");
return (new DownloadManager(
this.server,
this.gui,
this.log,
this.directoryManager.getActualStorageBinaryPathFor(ajob),
ajob.getRendererMD5(),
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
)).download();
return this.downloadChunks(ajob.getRendererChunks(), "Downloading Blender");
}
protected void removeSceneDirectory(Job ajob) {
@@ -863,17 +856,18 @@ import okhttp3.HttpUrl;
protected int prepareWorkingDirectory(Job ajob) {
int ret;
String rendererArchive = this.directoryManager.getCacheBinaryPathFor(ajob);
String rendererPath = ajob.getRendererDirectory();
File rendererPathFile = new File(rendererPath);
// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
log.error("Error while copying " + rendererArchive + " from shared downloads directory to working dir");
// chunk files are already downloaded, either on shared directory or cache directory
for (Chunk chunk: Utils.concatWithCollection(ajob.getRendererChunks(), ajob.getArchiveChunks())) {
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
this.gui.status("Copying chunk from common directory");
if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
}
}
}
if (!rendererPathFile.exists()) {
@@ -881,14 +875,12 @@ import okhttp3.HttpUrl;
rendererPathFile.mkdir();
this.gui.status("Extracting renderer");
this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + rendererArchive + " into " + rendererPath);
this.log.debug("Client::prepareWorkingDirectory Extracting renderer into " + rendererPath);
// unzip the archive
ret = Utils.unzipFileIntoDirectory(rendererArchive, rendererPath, null, log);
ret = Utils.unzipChunksIntoDirectory(ajob.getRendererChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()), rendererPath, null, log);
if (ret != 0) {
this.log.error(
"Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + rendererArchive + ", " + rendererPath + ") returned "
+ ret);
this.log.error("Client::prepareWorkingDirectory, error(1) with Utils.unzipChunksIntoDirectory(" + rendererPath + ") returned " + ret);
this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
return -1;
}
@@ -904,19 +896,6 @@ import okhttp3.HttpUrl;
String scenePath = ajob.getSceneDirectory();
File scenePathFile = new File(scenePath);
// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
for (Chunk chunk: ajob.getArchiveChunks()) {
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
this.gui.status("Copying chunk from common directory");
if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
}
}
}
/// download the chunks
if (!scenePathFile.exists()) {
// we create the directory
scenePathFile.mkdir();