Merge branch 'ref/download' into 'master'
Ref: split code, do not put everything on Client class See merge request sheepitrenderfarm/client!271
This commit is contained in:
Sheepit Renderfarm
@@ -37,7 +37,6 @@ import java.util.Locale;
|
|||||||
import java.util.Observable;
|
import java.util.Observable;
|
||||||
import java.util.Observer;
|
import java.util.Observer;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Random;
|
|
||||||
import java.util.Timer;
|
import java.util.Timer;
|
||||||
import java.util.TimerTask;
|
import java.util.TimerTask;
|
||||||
import java.util.concurrent.ArrayBlockingQueue;
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
@@ -63,7 +62,6 @@ import com.sheepit.client.exception.SheepItExceptionSessionDisabled;
|
|||||||
import com.sheepit.client.exception.SheepItExceptionSessionDisabledDenoisingNotSupported;
|
import com.sheepit.client.exception.SheepItExceptionSessionDisabledDenoisingNotSupported;
|
||||||
import com.sheepit.client.exception.SheepItServerDown;
|
import com.sheepit.client.exception.SheepItServerDown;
|
||||||
import com.sheepit.client.hardware.cpu.CPU;
|
import com.sheepit.client.hardware.cpu.CPU;
|
||||||
import com.sheepit.client.hardware.hwid.HWIdentifier;
|
|
||||||
import com.sheepit.client.os.OS;
|
import com.sheepit.client.os.OS;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -90,8 +88,6 @@ import okhttp3.HttpUrl;
|
|||||||
private boolean suspended;
|
private boolean suspended;
|
||||||
private boolean shuttingdown;
|
private boolean shuttingdown;
|
||||||
|
|
||||||
private int maxDownloadFileAttempts = 5;
|
|
||||||
|
|
||||||
private int uploadQueueSize;
|
private int uploadQueueSize;
|
||||||
private long uploadQueueVolume;
|
private long uploadQueueVolume;
|
||||||
private int noJobRetryIter;
|
private int noJobRetryIter;
|
||||||
@@ -865,7 +861,16 @@ import okhttp3.HttpUrl;
|
|||||||
int total = ajob_.getArchiveChunks().size();
|
int total = ajob_.getArchiveChunks().size();
|
||||||
for (int i = 0; i < total; i++) {
|
for (int i = 0; i < total; i++) {
|
||||||
Chunk chunk = ajob_.getArchiveChunks().get(i);
|
Chunk chunk = ajob_.getArchiveChunks().get(i);
|
||||||
Error.Type ret = this.downloadFile(ajob_, ajob_.getRequiredProjectChunkPath(chunk.getId()), chunk.getMd5(), String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId()), String.format(LOCALE, "chunk %d/%d", i + 1, total));
|
DownloadManager downloadManager = new DownloadManager(
|
||||||
|
this.server,
|
||||||
|
this.gui,
|
||||||
|
this.log,
|
||||||
|
String.format(LOCALE, "chunk %d/%d", i + 1, total),
|
||||||
|
ajob_.getRequiredProjectChunkPath(chunk.getId()),
|
||||||
|
chunk.getMd5(),
|
||||||
|
String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
|
||||||
|
);
|
||||||
|
Error.Type ret = downloadManager.download();
|
||||||
if (ret != Type.OK) {
|
if (ret != Type.OK) {
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
@@ -874,134 +879,15 @@ import okhttp3.HttpUrl;
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
|
protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
|
||||||
return this.downloadFile(ajob, ajob.getRequiredRendererArchivePath(), ajob.getRendererMD5(),
|
return (new DownloadManager(
|
||||||
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId()), "renderer");
|
this.server,
|
||||||
}
|
this.gui,
|
||||||
|
this.log,
|
||||||
private Error.Type downloadFile(Job ajob, String local_path, String md5_server, String url, String download_type) throws SheepItException {
|
"renderer",
|
||||||
File local_path_file = new File(local_path);
|
ajob.getRequiredRendererArchivePath(),
|
||||||
String update_ui = "Downloading " + download_type;
|
ajob.getRendererMD5(),
|
||||||
|
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
|
||||||
int remaining = 1800000; // 30 minutes max timeout
|
)).download();
|
||||||
|
|
||||||
try {
|
|
||||||
// If the client is using a shared cache then introduce some random delay to minimise race conditions on the partial file creation on multiple
|
|
||||||
// instances of a client (when started with a script or rendering a recently downloaded scene)
|
|
||||||
if (configuration.getSharedDownloadsDirectory() != null) {
|
|
||||||
Thread.sleep((new Random().nextInt(9) + 1) * 1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
// For a maximum of 30 minutes
|
|
||||||
do {
|
|
||||||
// if the binary or scene already exists in the cache
|
|
||||||
if (local_path_file.exists()) {
|
|
||||||
this.gui.status("Reusing cached " + download_type);
|
|
||||||
return Type.OK;
|
|
||||||
}
|
|
||||||
// if the binary or scene is being downloaded by another client
|
|
||||||
else if (new File(local_path + ".partial").exists()) {
|
|
||||||
// Wait and check every second for file download completion but only update the GUI every 10 seconds to minimise CPU load
|
|
||||||
if (remaining % 10000 == 0) {
|
|
||||||
this.gui.status(String.format("Another client is downloading the %s. Cancel in %dmin %ds",
|
|
||||||
download_type,
|
|
||||||
TimeUnit.MILLISECONDS.toMinutes(remaining),
|
|
||||||
TimeUnit.MILLISECONDS.toSeconds(remaining) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(remaining))
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
|
|
||||||
// detection by other concurrent clients and start downloading process
|
|
||||||
try {
|
|
||||||
File file = new File(local_path + ".partial");
|
|
||||||
file.createNewFile();
|
|
||||||
file.deleteOnExit(); // if the client crashes, the temporary file will be removed
|
|
||||||
} catch (IOException e) {
|
|
||||||
StringWriter sw = new StringWriter();
|
|
||||||
e.printStackTrace(new PrintWriter(sw));
|
|
||||||
this.log.error("Client::DownloadFile Unable to create .partial temp file for binary/scene " + local_path);
|
|
||||||
this.log.error("Client::DownloadFile Exception " + e + " stacktrace " + sw.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reduce 1 second the waiting time
|
|
||||||
Thread.sleep(1000);
|
|
||||||
remaining -= 1000;
|
|
||||||
} while (remaining > 0);
|
|
||||||
}
|
|
||||||
catch (InterruptedException e) {
|
|
||||||
log.debug("Error in the thread wait. Exception " + e.getMessage());
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
// If we have reached the timeout (30 minutes trying to download the client) delete the partial downloaded copy and try to download again
|
|
||||||
if (remaining <= 0) {
|
|
||||||
log.debug("ERROR while waiting for download to finish in another client. Deleting the partial file and downloading a fresh copy now!.");
|
|
||||||
new File(local_path + ".partial").delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.gui.status(String.format("Downloading %s", download_type));
|
|
||||||
|
|
||||||
// must download the archive
|
|
||||||
Error.Type ret = this.server.HTTPGetFile(url, local_path, this.gui, update_ui);
|
|
||||||
|
|
||||||
if (ret == Type.RENDERER_KILLED_BY_SERVER || ret == Type.RENDERER_KILLED_BY_USER_OVER_TIME || ret == Type.RENDERER_KILLED_BY_USER) {
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to check the download file even if a download error has occurred (MD5 file check will delete the file if partially downloaded)
|
|
||||||
boolean md5_check = this.checkFile(ajob, local_path, md5_server);
|
|
||||||
int attempts = 1;
|
|
||||||
|
|
||||||
while ((ret != Error.Type.OK || md5_check == false) && attempts < this.maxDownloadFileAttempts) {
|
|
||||||
if (ret != Error.Type.OK) {
|
|
||||||
this.gui.error(String.format("Unable to download %s (error %s). Retrying now", download_type, ret));
|
|
||||||
this.log.debug("Client::downloadFile problem with Server.HTTPGetFile (return: " + ret + ") removing local file (path: " + local_path + ")");
|
|
||||||
}
|
|
||||||
else if (md5_check == false) {
|
|
||||||
this.gui.error(String.format("Verification of downloaded %s has failed. Retrying now", download_type));
|
|
||||||
this.log.debug("Client::downloadFile problem with Client::checkFile mismatch on md5, removing local file (path: " + local_path + ")");
|
|
||||||
}
|
|
||||||
local_path_file.delete();
|
|
||||||
|
|
||||||
this.log.debug("Client::downloadFile failed, let's try again (" + (attempts + 1) + "/" + this.maxDownloadFileAttempts + ") ...");
|
|
||||||
|
|
||||||
ret = this.server.HTTPGetFile(url, local_path, this.gui, update_ui);
|
|
||||||
|
|
||||||
md5_check = this.checkFile(ajob, local_path, md5_server);
|
|
||||||
attempts++;
|
|
||||||
|
|
||||||
if ((ret != Error.Type.OK || md5_check == false) && attempts >= this.maxDownloadFileAttempts) {
|
|
||||||
this.log.debug("Client::downloadFile failed after " + this.maxDownloadFileAttempts + " attempts, removing local file (path: " + local_path
|
|
||||||
+ "), stopping...");
|
|
||||||
local_path_file.delete();
|
|
||||||
return Type.DOWNLOAD_FILE;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Type.OK;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean checkFile(Job ajob, String local_path, String md5_server) {
|
|
||||||
File local_path_file = new File(local_path);
|
|
||||||
|
|
||||||
if (local_path_file.exists() == false) {
|
|
||||||
this.log.error("Client::checkFile cannot check md5 on a nonexistent file (path: " + local_path + ")");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
String md5_local = Utils.md5(local_path);
|
|
||||||
|
|
||||||
if (md5_local.equals(md5_server) == false) {
|
|
||||||
this.log.error(
|
|
||||||
"Client::checkFile mismatch on md5 local: '" + md5_local + "' server: '" + md5_server + "' (local size: " + new File(local_path).length()
|
|
||||||
+ ")");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void removeSceneDirectory(Job ajob) {
|
protected void removeSceneDirectory(Job ajob) {
|
||||||
|
|||||||
194
src/main/java/com/sheepit/client/DownloadManager.java
Normal file
194
src/main/java/com/sheepit/client/DownloadManager.java
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2023 Laurent CLOUET
|
||||||
|
* Author Laurent CLOUET <laurent.clouet@nopnop.net>
|
||||||
|
*
|
||||||
|
* This program is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU General Public License
|
||||||
|
* as published by the Free Software Foundation; version 2
|
||||||
|
* of the License.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
package com.sheepit.client;
|
||||||
|
|
||||||
|
import com.sheepit.client.exception.SheepItException;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class DownloadManager {
|
||||||
|
private static int maxDownloadFileAttempts = 5;
|
||||||
|
|
||||||
|
// global objects
|
||||||
|
private Server server;
|
||||||
|
private Gui gui;
|
||||||
|
private Log log;
|
||||||
|
|
||||||
|
// task specific objects
|
||||||
|
private String gui_text; // what do display on the gui
|
||||||
|
private String local_target;
|
||||||
|
private String md5; // expected md5 of the file, for check purpose
|
||||||
|
private String remote; // remote url
|
||||||
|
|
||||||
|
public DownloadManager(Server server, Gui gui, Log log, String gui_text, String local_target, String md5, String remote) {
|
||||||
|
this.server = server;
|
||||||
|
this.gui = gui;
|
||||||
|
this.gui_text = gui_text;
|
||||||
|
this.log = log;
|
||||||
|
this.local_target = local_target;
|
||||||
|
this.md5 = md5;
|
||||||
|
this.remote = remote;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Error.Type download() throws SheepItException {
|
||||||
|
File local_path_file = new File(this.local_target);
|
||||||
|
|
||||||
|
int remaining = 1800000; // 30 minutes max timeout
|
||||||
|
|
||||||
|
try {
|
||||||
|
// For a maximum of 30 minutes
|
||||||
|
do {
|
||||||
|
// if the binary or scene already exists in the cache
|
||||||
|
if (local_path_file.exists()) {
|
||||||
|
this.gui.status("Reusing cached " + this.gui_text);
|
||||||
|
return Error.Type.OK;
|
||||||
|
}
|
||||||
|
// if the binary or scene is being downloaded by another client
|
||||||
|
else if (this.lockExists()) {
|
||||||
|
// Wait and check every second for file download completion but only update the GUI every 10 seconds to minimise CPU load
|
||||||
|
if (remaining % 10000 == 0) {
|
||||||
|
this.gui.status(String.format("Another client is downloading the %s. Cancel in %dmin %ds",
|
||||||
|
this.gui_text,
|
||||||
|
TimeUnit.MILLISECONDS.toMinutes(remaining),
|
||||||
|
TimeUnit.MILLISECONDS.toSeconds(remaining) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(remaining))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// The file doesn't yet exist not is being downloaded by another client, so immediately create the file with zero bytes to allow early
|
||||||
|
// detection by other concurrent clients and start downloading process
|
||||||
|
this.createLock();
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait about 1 second on average
|
||||||
|
int wait = 1 + (new Random()).nextInt(2000);
|
||||||
|
Thread.sleep(wait);
|
||||||
|
remaining -= wait;
|
||||||
|
} while (remaining > 0);
|
||||||
|
}
|
||||||
|
catch (InterruptedException e) {
|
||||||
|
log.debug("Error in the thread wait. Exception " + e.getMessage());
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
// If we have reached the timeout (30 minutes trying to download the client) delete the partial downloaded copy and try to download again
|
||||||
|
if (remaining <= 0) {
|
||||||
|
log.debug("ERROR while waiting for download to finish in another client. Deleting the partial file and downloading a fresh copy now!.");
|
||||||
|
this.removeLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.gui.status(String.format("Downloading %s", this.gui_text));
|
||||||
|
|
||||||
|
return this.downloadActual();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Error.Type downloadActual() throws SheepItException {
|
||||||
|
String update_ui = "Downloading " + this.gui_text;
|
||||||
|
|
||||||
|
// must download the archive
|
||||||
|
Error.Type ret = this.server.HTTPGetFile(this.remote, this.local_target, this.gui, update_ui);
|
||||||
|
|
||||||
|
if (ret == Error.Type.RENDERER_KILLED_BY_SERVER || ret == Error.Type.RENDERER_KILLED_BY_USER_OVER_TIME || ret == Error.Type.RENDERER_KILLED_BY_USER) {
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to check the download file even if a download error has occurred (MD5 file check will delete the file if partially downloaded)
|
||||||
|
boolean md5_check = this.check();
|
||||||
|
int attempts = 1;
|
||||||
|
|
||||||
|
while ((ret != Error.Type.OK || md5_check == false) && attempts < this.maxDownloadFileAttempts) {
|
||||||
|
if (ret != Error.Type.OK) {
|
||||||
|
this.gui.error(String.format("Unable to download %s (error %s). Retrying now", this.gui_text, ret));
|
||||||
|
this.log.debug("DownloadManager::downloadActual problem with Server.HTTPGetFile (return: " + ret + ") removing local file (path: " + this.local_target + ")");
|
||||||
|
}
|
||||||
|
else if (md5_check == false) {
|
||||||
|
this.gui.error(String.format("Verification of downloaded %s has failed. Retrying now", this.gui_text));
|
||||||
|
this.log.debug("DownloadManager::downloadActual problem with Client::checkFile mismatch on md5, removing local file (path: " + this.local_target + ")");
|
||||||
|
}
|
||||||
|
|
||||||
|
(new File(this.local_target)).delete();
|
||||||
|
|
||||||
|
this.log.debug("DownloadManager::downloadActual failed, let's try again (" + (attempts + 1) + "/" + this.maxDownloadFileAttempts + ") ...");
|
||||||
|
|
||||||
|
String partial_target = this.local_target + ".partial";
|
||||||
|
ret = this.server.HTTPGetFile(this.remote, partial_target, this.gui, update_ui);
|
||||||
|
|
||||||
|
md5_check = this.check();
|
||||||
|
attempts++;
|
||||||
|
|
||||||
|
if ((ret != Error.Type.OK || md5_check == false) && attempts >= this.maxDownloadFileAttempts) {
|
||||||
|
this.log.debug("DownloadManager::downloadActual failed after " + this.maxDownloadFileAttempts + " attempts, removing local file (path: " + this.local_target + "), stopping...");
|
||||||
|
// local_path_file.delete();
|
||||||
|
return Error.Type.DOWNLOAD_FILE;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return (new File (partial_target)).renameTo(new File(this.local_target)) ? Error.Type.OK : Error.Type.DOWNLOAD_FILE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Error.Type.OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean check() {
|
||||||
|
File local_path_file = new File(this.local_target);
|
||||||
|
|
||||||
|
if (local_path_file.exists() == false) {
|
||||||
|
this.log.error("DownloadManager::check cannot check md5 on a nonexistent file (path: " + this.local_target + ")");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
String md5_local = Utils.md5(this.local_target);
|
||||||
|
|
||||||
|
if (md5_local.equals(this.md5) == false) {
|
||||||
|
this.log.error("DownloadManager::check mismatch on md5 local: '" + md5_local + "' server: '" + this.md5 + "' (local size: " + new File(this.local_target).length() + ")");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean lockExists() {
|
||||||
|
return new File(this.local_target + ".partial").exists();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createLock() {
|
||||||
|
try {
|
||||||
|
File file = new File(this.local_target + ".partial");
|
||||||
|
file.createNewFile();
|
||||||
|
file.deleteOnExit(); // if the client crashes, the temporary file will be removed
|
||||||
|
}
|
||||||
|
catch (IOException e) {
|
||||||
|
StringWriter sw = new StringWriter();
|
||||||
|
e.printStackTrace(new PrintWriter(sw));
|
||||||
|
this.log.error("DownloadManager::createLock Unable to create .partial temp file for binary/scene " + this.local_target);
|
||||||
|
this.log.error("DownloadManager::createLock Exception " + e + " stacktrace " + sw.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void removeLock() {
|
||||||
|
new File(this.local_target + ".partial").delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user