Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ref: move directories handle to a specific class (outside of Client class)

Browse Source
This commit is contained in:
Sheepit Renderfarm
2023-12-19 14:35:24 +00:00
parent b4d7ee5edb
commit 3e850ad6e5
3 changed files with 123 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/com/sheepit/client/Client.java
View File

@@ -24,11 +24,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@@ -66,6 +61,7 @@ import okhttp3.HttpUrl;
@Data public class Client {
public static final int MIN_JOB_ID = 20; //to distinguish between actual jobs and test frames
private static final Locale LOCALE = Locale.ENGLISH;
private DirectoryManager directoryManager;
private Gui gui;
private Server server;
private Configuration configuration;
@@ -92,6 +88,7 @@ import okhttp3.HttpUrl;
this.server = new Server(url_, this.configuration, this);
this.log = Log.getInstance(this.configuration);
this.gui = gui_;
this.directoryManager = new DirectoryManager(this.configuration, this.log);
this.renderingJob = null;
this.previousJob = null;
this.jobsToValidate = new ArrayBlockingQueue<QueuedJob>(5);
@@ -793,7 +790,7 @@ import okhttp3.HttpUrl;
this.gui,
this.log,
String.format(LOCALE, "chunk %d/%d", i + 1, total),
ajob_.getRequiredProjectChunkPath(chunk.getId()),
this.directoryManager.getActualStoragePathFor(chunk),
chunk.getMd5(),
String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
);
@@ -811,7 +808,7 @@ import okhttp3.HttpUrl;
this.gui,
this.log,
"renderer",
ajob.getRequiredRendererArchivePath(),
this.directoryManager.getActualStorageBinaryPathFor(ajob),
ajob.getRendererMD5(),
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
)).download();
@@ -823,15 +820,18 @@ import okhttp3.HttpUrl;
protected int prepareWorkingDirectory(Job ajob) {
int ret;
String bestRendererArchive = ajob.getRequiredRendererArchivePath();
String renderer_archive = ajob.getRendererArchivePath();
String renderer_archive = this.directoryManager.getCacheBinaryPathFor(ajob);
String renderer_path = ajob.getRendererDirectory();
File renderer_path_file = new File(renderer_path);
if (!new File(renderer_archive).exists()) {
// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + bestRendererArchive + " into " + renderer_archive);
copySharedChunk(bestRendererArchive, renderer_archive);
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
log.error("Error while copying " + renderer_archive + " from shared downloads directory to working dir");
}
}
if (!renderer_path_file.exists()) {
@@ -862,11 +862,15 @@ import okhttp3.HttpUrl;
String scene_path = ajob.getSceneDirectory();
File scene_path_file = new File(scene_path);
// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
for (Chunk chunk: ajob.getArchiveChunks()) {
if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) {
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
this.gui.status("Copying chunk from common directory");
copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId()));
if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
}
}
}
/// download the chunks
@@ -881,7 +885,7 @@ import okhttp3.HttpUrl;
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()),
ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
scene_path,
ajob.getPassword(),
log);
@@ -895,28 +899,6 @@ import okhttp3.HttpUrl;
return 0;
}
private void copySharedChunk(String existingArchive, String targetArchive) {
Path existingArchivePath = Paths.get(existingArchive);
Path targetArchivePath = Paths.get(targetArchive);
try {
try {
Files.createLink(targetArchivePath, existingArchivePath);
log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath);
}
catch (UnsupportedOperationException // underlying file system does not support hard-linking
| FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported
| SecurityException // user is not allowed to create hard-links
ignore) {
// Creating hardlinks might not be supported on some filesystems
log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
}
}
catch (IOException e) {
this.gui.error("Error while copying " + existingArchive + " from shared downloads directory to working dir");
}
}
protected Error.Type confirmJob(Job ajob, int checkpoint) {
String url_real = String.format(LOCALE, "%s&rendertime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(),
ajob.getProcessRender().getPeakMemoryUsed());