Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ref: split code, do not put everything on Client class

Browse Source
This commit is contained in:
Sheepit Renderfarm
2023-12-07 12:27:53 +00:00
parent 49e4baaf93
commit 047312f8b9
2 changed files with 213 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
src/main/java/com/sheepit/client/Client.java
View File

@@ -37,7 +37,6 @@ import java.util.Locale;
import java.util.Observable;
import java.util.Observer;
import java.util.Optional;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ArrayBlockingQueue;
@@ -63,7 +62,6 @@ import com.sheepit.client.exception.SheepItExceptionSessionDisabled;
import com.sheepit.client.exception.SheepItExceptionSessionDisabledDenoisingNotSupported;
import com.sheepit.client.exception.SheepItServerDown;
import com.sheepit.client.hardware.cpu.CPU;
import com.sheepit.client.hardware.hwid.HWIdentifier;
import com.sheepit.client.os.OS;
import lombok.AllArgsConstructor;
@@ -90,8 +88,6 @@ import okhttp3.HttpUrl;
private boolean suspended;
private boolean shuttingdown;
private int maxDownloadFileAttempts = 5;
private int uploadQueueSize;
private long uploadQueueVolume;
private int noJobRetryIter;
@@ -865,7 +861,16 @@ import okhttp3.HttpUrl;
int total = ajob_.getArchiveChunks().size();
for (int i = 0; i < total; i++) {
Chunk chunk = ajob_.getArchiveChunks().get(i);
Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), String.format(LOCALE, "chunk %d/%d", i + 1, total));
DownloadManager downloadManager = new DownloadManager(
this.server,
this.gui,
this.log,
String.format(LOCALE, "chunk %d/%d", i + 1, total),
ajob_.getRequiredProjectChunkPath(chunk.getId()),
chunk.getMd5(),
String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
);
Error.Type ret = downloadManager.download();
if (ret != Type.OK) {
return ret;
}
@@ -874,134 +879,15 @@ import okhttp3.HttpUrl;
}
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer");
}
private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException {
File local_path_file = new File(local_path);
String update_ui = "Downloading " + download_type;
int remaining = 1800000; // 30 minutes max timeout
try {
// If the client is using a shared cache then introduce some random delay to minimise race conditions on the partial file creation on multiple
// instances of a client (when started with a script or rendering a recently downloaded scene)
if (configuration.getSharedDownloadsDirectory() != null) {
Thread.sleep((new Random().nextInt(9) + 1) * 1000);
}
// For a maximum of 30 minutes
do {
// if the binary or scene already exists in the cache
if (local_path_file.exists()) {
this.gui.status("Reusing cached " + download_type);
return Type.OK;
}
// if the binary or scene is being downloaded by another client
else if (new File(local_path + ".partial").exists()) {
// Wait and check every second for file download completion but only update the GUI every 10 seconds to minimise CPU load
if (remaining % 10000 == 0) {
this.gui.status(String.format("Another client is downloading the %s. Cancel in %dmin %ds",
download_type,
TimeUnit.MILLISECONDS.toMinutes(remaining),
TimeUnit.MILLISECONDS.toSeconds(remaining) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(remaining))
));
}
}
else {
// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
// detection by other concurrent clients and start downloading process
try {
File file = new File(local_path + ".partial");
file.createNewFile();
file.deleteOnExit(); // if the client crashes, the temporary file will be removed
} catch (IOException e) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
this.log.error("Client::DownloadFile Unable to create .partial temp file for binary/scene " + local_path);
this.log.error("Client::DownloadFile Exception " + e + " stacktrace " + sw.toString());
}
break;
}
// Reduce 1 second the waiting time
Thread.sleep(1000);
remaining -= 1000;
} while (remaining > 0);
}
catch (InterruptedException e) {
log.debug("Error in the thread wait. Exception " + e.getMessage());
}
finally {
// If we have reached the timeout (30 minutes trying to download the client) delete the partial downloaded copy and try to download again
if (remaining <= 0) {
log.debug("ERROR while waiting for download to finish in another client. Deleting the partial file and downloading a fresh copy now!.");
new File(local_path + ".partial").delete();
}
}
this.gui.status(String.format("Downloading %s", download_type));
// must download the archive
Error.Type ret = this.server.HTTPGetFile(url, local_path, this.gui, update_ui);
if (ret == Type.RENDERER_KILLED_BY_SERVER || ret == Type.RENDERER_KILLED_BY_USER_OVER_TIME || ret == Type.RENDERER_KILLED_BY_USER) {
return ret;
}
// Try to check the download file even if a download error has occurred (MD5 file check will delete the file if partially downloaded)
boolean md5_check = this.checkFile(ajob, local_path, md5_server);
int attempts = 1;
while ((ret != Error.Type.OK || md5_check == false) && attempts < this.maxDownloadFileAttempts) {
if (ret != Error.Type.OK) {
this.gui.error(String.format("Unable to download %s (error %s). Retrying now", download_type, ret));
this.log.debug("Client::downloadFile problem with Server.HTTPGetFile (return: " + ret + ") removing local file (path: " + local_path + ")");
}
else if (md5_check == false) {
this.gui.error(String.format("Verification of downloaded %s has failed. Retrying now", download_type));
this.log.debug("Client::downloadFile problem with Client::checkFile mismatch on md5, removing local file (path: " + local_path + ")");
}
local_path_file.delete();
this.log.debug("Client::downloadFile failed, let's try again (" + (attempts + 1) + "/" + this.maxDownloadFileAttempts + ") ...");
ret = this.server.HTTPGetFile(url, local_path, this.gui, update_ui);
md5_check = this.checkFile(ajob, local_path, md5_server);
attempts++;
if ((ret != Error.Type.OK || md5_check == false) && attempts >= this.maxDownloadFileAttempts) {
this.log.debug("Client::downloadFile failed after " + this.maxDownloadFileAttempts + " attempts, removing local file (path: " + local_path
+ "), stopping...");
local_path_file.delete();
return Type.DOWNLOAD_FILE;
}
}
return Type.OK;
}
private boolean checkFile(Job ajob, String local_path, String md5_server) {
File local_path_file = new File(local_path);
if (local_path_file.exists() == false) {
this.log.error("Client::checkFile cannot check md5 on a nonexistent file (path: " + local_path + ")");
return false;
}
String md5_local = Utils.md5(local_path);
if (md5_local.equals(md5_server) == false) {
this.log.error(
"Client::checkFile mismatch on md5 local: '" + md5_local + "' server: '" + md5_server + "' (local size: " + new File(local_path).length()
+ ")");
return false;
}
return true;
return (new DownloadManager(
this.server,
this.gui,
this.log,
"renderer",
ajob.getRequiredRendererArchivePath(),
ajob.getRendererMD5(),
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
)).download();
}
protected void removeSceneDirectory(Job ajob) {