Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: add static analytic of code

Browse Source
This commit is contained in:
Laurent Clouet
2024-06-03 14:02:30 +00:00
parent c966a1a032
commit 4bce6409e3
52 changed files with 249 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
src/main/java/com/sheepit/client/Client.java
View File

@@ -91,15 +91,15 @@ import okhttp3.HttpUrl;
private long uploadQueueVolume;
private int noJobRetryIter;
public Client(Gui gui_, Configuration configuration, String url_) {
public Client(Gui gui, Configuration configuration, String url) {
this.configuration = configuration;
this.server = new Server(url_, this.configuration, this);
this.server = new Server(url, this.configuration, this);
this.log = Log.getInstance(this.configuration);
this.gui = gui_;
this.gui = gui;
this.directoryManager = new DirectoryManager(this.configuration, this.log);
this.renderingJob = null;
this.previousJob = null;
this.jobsToValidate = new ArrayBlockingQueue<QueuedJob>(5);
this.jobsToValidate = new ArrayBlockingQueue<>(5);
this.isValidatingJob = false;
this.disableErrorSending = false;
@@ -115,7 +115,7 @@ import okhttp3.HttpUrl;
this.sessionStarted = false;
}
public String toString() {
@Override public String toString() {
return String.format("Client (configuration %s, server %s)", this.configuration, this.server);
}
@@ -155,7 +155,7 @@ import okhttp3.HttpUrl;
shuttingdown = true;
log.debug("Initiating the computer's shutting down process");
if (configuration.getShutdownMode().equals("wait")) {
if ("wait".equals(configuration.getShutdownMode())) {
// Soft stop. Complete current render (if any), finish uploading frames and then shutdown the computer
askForStop();
}
@@ -183,13 +183,9 @@ import okhttp3.HttpUrl;
this.server.start(); // for staying alive
// create a thread which will send the frame
Runnable runnable_sender = new Runnable() {
public void run() {
senderLoop();
}
};
Thread thread_sender = new Thread(runnable_sender);
thread_sender.start();
Runnable runnableSender = this::senderLoop;
Thread threadSender = new Thread(runnableSender);
threadSender.start();
do {
while (this.running) {
@@ -207,12 +203,12 @@ import okhttp3.HttpUrl;
step = this.log.newCheckPoint();
try {
Calendar next_request = this.nextJobRequest();
if (next_request != null) {
Calendar nextRequest = this.nextJobRequest();
if (nextRequest != null) {
// wait
Date now = new Date();
this.gui.status(String.format("Waiting until %tR before requesting job", next_request));
long wait = next_request.getTimeInMillis() - now.getTime();
this.gui.status(String.format("Waiting until %tR before requesting job", nextRequest));
long wait = nextRequest.getTimeInMillis() - now.getTime();
if (wait < 0) {
// it means the client has to wait until the next day
wait += 24 * 3600 * 1000;
@@ -247,12 +243,12 @@ import okhttp3.HttpUrl;
else {
this.startTime = new Date().getTime(); // reset start session time because the server did it
try {
Calendar next_request = this.nextJobRequest();
if (next_request != null) {
Calendar nextRequest = this.nextJobRequest();
if (nextRequest != null) {
// wait
Date now = new Date();
this.gui.status(String.format("Waiting until %tR before requesting job", next_request));
long timeToSleep = next_request.getTimeInMillis() - now.getTime();
this.gui.status(String.format("Waiting until %tR before requesting job", nextRequest));
long timeToSleep = nextRequest.getTimeInMillis() - now.getTime();
this.activeSleep(timeToSleep);
}
@@ -276,10 +272,10 @@ import okhttp3.HttpUrl;
// SheepItServerDown
// SheepItExceptionBadResponseFromServer
int time_sleep = e.getWaitDuration();
this.gui.status(String.format(e.getHumanText(), new Date(new Date().getTime() + time_sleep)));
int timeSleep = e.getWaitDuration();
this.gui.status(String.format(e.getHumanText(), new Date(new Date().getTime() + timeSleep)));
if (this.activeSleep(time_sleep) == false) {
if (this.activeSleep(timeSleep) == false) {
return -3;
}
this.log.removeCheckPoint(step);
@@ -297,13 +293,13 @@ import okhttp3.HttpUrl;
}
if (this.renderingJob == null) { // no job
int[] retrySchemeInMilliSeconds = { 300000, 480000, 720000, 900000, 1200000 }; // 5, 8, 12, 15 and 20 minutes
int[] retrySchemeInMilliSeconds = { 300_000, 480_000, 720_000, 900_000, 1_200_000 }; // 5, 8, 12, 15 and 20 minutes
int time_sleep = retrySchemeInMilliSeconds[(this.noJobRetryIter < retrySchemeInMilliSeconds.length) ?
int timeSleep = retrySchemeInMilliSeconds[(this.noJobRetryIter < retrySchemeInMilliSeconds.length) ?
this.noJobRetryIter++ :
(retrySchemeInMilliSeconds.length - 1)];
this.gui.status(String.format("No job available. Will try again at %tR", new Date(new Date().getTime() + time_sleep)));
if (this.activeSleep(time_sleep) == false) {
this.gui.status(String.format("No job available. Will try again at %tR", new Date(new Date().getTime() + timeSleep)));
if (this.activeSleep(timeSleep) == false) {
return -3;
}
this.log.removeCheckPoint(step);
@@ -317,10 +313,10 @@ import okhttp3.HttpUrl;
ret = this.work(this.renderingJob);
if (ret == Error.Type.NO_SPACE_LEFT_ON_DEVICE || ret == Error.Type.PATH_INVALID || ret == Error.Type.NO_WRITE_PERMISSION ) {
Job frame_to_reset = this.renderingJob; // copy it because the sendError will take ~5min to execute
Job frameToReset = this.renderingJob; // copy it because the sendError will take ~5min to execute
this.renderingJob = null;
this.gui.error(Error.humanString(ret));
this.sendError(step, frame_to_reset, ret);
this.sendError(step, frameToReset, ret);
this.log.removeCheckPoint(step);
return -50;
}
@@ -433,7 +429,7 @@ import okhttp3.HttpUrl;
try {
this.server.HTTPRequest(this.server.getPage("logout"));
}
catch (IOException e) {
catch (IOException ignored) {
// nothing to do: if the logout failed that's ok
}
}
@@ -562,11 +558,11 @@ import okhttp3.HttpUrl;
}
}
protected void sendError(int step_) {
this.sendError(step_, null, null);
protected void sendError(int step) {
this.sendError(step, null, null);
}
protected void sendError(int step_, Job job_to_reset_, Error.Type error) {
protected void sendError(int step, Job jobToReset, Error.Type error) {
if (this.disableErrorSending) {
this.log.debug("Error sending is disabled, do not send log");
return;
@@ -574,10 +570,10 @@ import okhttp3.HttpUrl;
this.log.debug("Sending error to server (type: " + error + ")");
try {
File temp_file = File.createTempFile("farm_", ".txt");
temp_file.createNewFile();
temp_file.deleteOnExit();
FileOutputStream writer = new FileOutputStream(temp_file);
File tempFile = File.createTempFile("farm_", ".txt");
tempFile.createNewFile();
tempFile.deleteOnExit();
FileOutputStream writer = new FileOutputStream(tempFile);
// Create a header with the information summarised for easier admin error analysis
Configuration conf = this.configuration;
@@ -594,9 +590,9 @@ import okhttp3.HttpUrl;
}
logHeader.append("====================================================================================================\n");
if (job_to_reset_ != null) {
logHeader.append(String.format("Project ::: %s\n", job_to_reset_.getName()))
.append(String.format("Project id: %s frame: %s\n", job_to_reset_.getId(), job_to_reset_.getFrameNumber())).append(String.format("blender ::: %s\n\n", job_to_reset_.getBlenderLongVersion())).append(String.format("ERROR Type :: %s\n", error));
if (jobToReset != null) {
logHeader.append(String.format("Project ::: %s\n", jobToReset.getName()))
.append(String.format("Project id: %s frame: %s\n", jobToReset.getId(), jobToReset.getFrameNumber())).append(String.format("blender ::: %s\n\n", jobToReset.getBlenderLongVersion())).append(String.format("ERROR Type :: %s\n", error));
}
else {
logHeader.append("Project ::: No project allocated.\n")
@@ -607,7 +603,7 @@ import okhttp3.HttpUrl;
// Insert the info at the beginning of the error log
writer.write(logHeader.toString().getBytes());
Optional<ArrayList<String>> logs = this.log.getForCheckPoint(step_);
Optional<ArrayList<String>> logs = this.log.getForCheckPoint(step);
if (logs.isPresent()) {
for (String line : logs.get()) {
writer.write(line.getBytes());
@@ -618,14 +614,14 @@ import okhttp3.HttpUrl;
writer.close();
HttpUrl.Builder remoteURL = HttpUrl.parse(this.server.getPage("error")).newBuilder();
remoteURL.addQueryParameter("type", error == null ? "" : Integer.toString(error.getValue()));
if (job_to_reset_ != null) {
remoteURL.addQueryParameter("frame", job_to_reset_.getFrameNumber());
remoteURL.addQueryParameter("job", job_to_reset_.getId());
remoteURL.addQueryParameter("render_time", Integer.toString(job_to_reset_.getProcessRender().getRenderDuration()));
remoteURL.addQueryParameter("memoryused", Long.toString(job_to_reset_.getProcessRender().getPeakMemoryUsed()));
if (jobToReset != null) {
remoteURL.addQueryParameter("frame", jobToReset.getFrameNumber());
remoteURL.addQueryParameter("job", jobToReset.getId());
remoteURL.addQueryParameter("render_time", Integer.toString(jobToReset.getProcessRender().getRenderDuration()));
remoteURL.addQueryParameter("memoryused", Long.toString(jobToReset.getProcessRender().getPeakMemoryUsed()));
}
this.server.HTTPSendFile(remoteURL.build().toString(), temp_file.getAbsolutePath(), step_, this.gui);
temp_file.delete();
this.server.HTTPSendFile(remoteURL.build().toString(), tempFile.getAbsolutePath(), step, this.gui);
tempFile.delete();
}
catch (Exception e) {
StringWriter sw = new StringWriter();
@@ -735,26 +731,26 @@ import okhttp3.HttpUrl;
}
}
final File scene_file = new File(ajob.getScenePath());
File renderer_file = new File(ajob.getRendererPath());
final File sceneFile = new File(ajob.getScenePath());
File rendererFile = new File(ajob.getRendererPath());
if (scene_file.exists() == false) {
if (sceneFile.exists() == false) {
gui.setRenderingProjectName("");
for (String logline : configuration.filesystemHealthCheck()) {
log.debug(logline);
}
this.log.error("Client::work job preparation failed (scene file '" + scene_file.getAbsolutePath()
this.log.error("Client::work job preparation failed (scene file '" + sceneFile.getAbsolutePath()
+ "' does not exist), cleaning directory in hope to recover");
this.configuration.cleanWorkingDirectory();
return Error.Type.MISSING_SCENE;
}
if (renderer_file.exists() == false) {
if (rendererFile.exists() == false) {
gui.setRenderingProjectName("");
for (String logline : configuration.filesystemHealthCheck()) {
log.debug(logline);
}
this.log.error("Client::work job preparation failed (renderer file '" + renderer_file.getAbsolutePath()
this.log.error("Client::work job preparation failed (renderer file '" + rendererFile.getAbsolutePath()
+ "' does not exist), cleaning directory in hope to recover");
this.configuration.cleanWorkingDirectory();
return Error.Type.MISSING_RENDERER;
@@ -764,7 +760,7 @@ import okhttp3.HttpUrl;
@Override public void update(Observable observable, Object o) {
// only remove the .blend since it's most important data
// and it's the only file we are sure will not be needed anymore
scene_file.delete();
sceneFile.delete();
}
};
@@ -782,8 +778,8 @@ import okhttp3.HttpUrl;
return err;
}
protected Error.Type downloadSceneFile(Job ajob_) throws SheepItException {
int total = ajob_.getArchiveChunks().size();
protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
int total = ajob.getArchiveChunks().size();
int threads = Math.max(1, Math.min(total, 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
ExecutorService executor = Executors.newFixedThreadPool(threads);
@@ -792,9 +788,8 @@ import okhttp3.HttpUrl;
this.gui.getDownloadProgress().reset("Downloading project");
for (int i = 0; i < total; i++) {
Chunk chunk = ajob_.getArchiveChunks().get(i);
Chunk chunk = ajob.getArchiveChunks().get(i);
int finalI = i;
Callable<Type> downloadTask = () -> {
DownloadManager downloadManager = new DownloadManager(
this.server,
@@ -804,9 +799,7 @@ import okhttp3.HttpUrl;
chunk.getMd5(),
String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
);
Type ret = null;
ret = downloadManager.download();
return ret;
return downloadManager.download();
};
tasks.add(downloadTask);
@@ -852,31 +845,31 @@ import okhttp3.HttpUrl;
protected int prepareWorkingDirectory(Job ajob) {
int ret;
String renderer_archive = this.directoryManager.getCacheBinaryPathFor(ajob);
String renderer_path = ajob.getRendererDirectory();
File renderer_path_file = new File(renderer_path);
String rendererArchive = this.directoryManager.getCacheBinaryPathFor(ajob);
String rendererPath = ajob.getRendererDirectory();
File rendererPathFile = new File(rendererPath);
// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
log.error("Error while copying " + renderer_archive + " from shared downloads directory to working dir");
log.error("Error while copying " + rendererArchive + " from shared downloads directory to working dir");
}
}
if (!renderer_path_file.exists()) {
if (!rendererPathFile.exists()) {
// we create the directory
renderer_path_file.mkdir();
rendererPathFile.mkdir();
this.gui.status("Extracting renderer");
this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + renderer_archive + " into " + renderer_path);
this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + rendererArchive + " into " + rendererPath);
// unzip the archive
ret = Utils.unzipFileIntoDirectory(renderer_archive, renderer_path, null, log);
ret = Utils.unzipFileIntoDirectory(rendererArchive, rendererPath, null, log);
if (ret != 0) {
this.log.error(
"Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + renderer_archive + ", " + renderer_path + ") returned "
"Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + rendererArchive + ", " + rendererPath + ") returned "
+ ret);
this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
return -1;
@@ -890,8 +883,8 @@ import okhttp3.HttpUrl;
}
}
String scene_path = ajob.getSceneDirectory();
File scene_path_file = new File(scene_path);
String scenePath = ajob.getSceneDirectory();
File scenePathFile = new File(scenePath);
// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
for (Chunk chunk: ajob.getArchiveChunks()) {
@@ -906,18 +899,18 @@ import okhttp3.HttpUrl;
/// download the chunks
if (!scene_path_file.exists()) {
if (!scenePathFile.exists()) {
// we create the directory
scene_path_file.mkdir();
scenePathFile.mkdir();
this.gui.status("Extracting project");
this.log.debug("Client::prepareWorkingDirectory Extracting project into " + scene_path);
this.log.debug("Client::prepareWorkingDirectory Extracting project into " + scenePath);
// unzip the archive
Instant startUnzip = Instant.now();
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
scene_path,
scenePath,
ajob.getPassword(),
log);
@@ -936,22 +929,22 @@ import okhttp3.HttpUrl;
}
protected Error.Type confirmJob(Job ajob, int checkpoint) {
String url_real = String.format(LOCALE, "%s&rendertime=%d&preptime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(), ajob.getProcessRender().getScenePrepDuration(),
String urlReal = String.format(LOCALE, "%s&rendertime=%d&preptime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(), ajob.getProcessRender().getScenePrepDuration(),
ajob.getProcessRender().getPeakMemoryUsed());
if (ajob.getSpeedSamplesRendered() > 0.0) {
url_real += String.format(LOCALE, "&speedsamples=%s", ajob.getSpeedSamplesRendered());
urlReal += String.format(LOCALE, "&speedsamples=%s", ajob.getSpeedSamplesRendered());
}
this.log.debug(checkpoint, "Client::confirmeJob url " + url_real);
this.log.debug(checkpoint, "Client::confirmeJob url " + urlReal);
this.log.debug(checkpoint, "path frame " + ajob.getOutputImagePath());
this.isValidatingJob = true;
int max_try = 3;
int timeToSleep = 22000;
ServerCode ret = ServerCode.UNKNOWN;
int maxTries = 3;
int timeToSleep = 22_000;
ServerCode ret;
Type confirmJobReturnCode = Error.Type.OK;
retryLoop:
for (int nb_try = 0; nb_try < max_try; nb_try++) {
if (nb_try >= 1) {
for (int nbTry = 0; nbTry < maxTries; nbTry++) {
if (nbTry >= 1) {
// sleep before retrying
this.log.debug(checkpoint, "Sleep for " + timeToSleep / 1000 + "s before trying to re-upload the frame, previous error: "+ confirmJobReturnCode);
try {
@@ -963,7 +956,7 @@ import okhttp3.HttpUrl;
timeToSleep *= 2; // exponential backoff
}
ret = this.server.HTTPSendFile(url_real, ajob.getOutputImagePath(), checkpoint, this.gui);
ret = this.server.HTTPSendFile(urlReal, ajob.getOutputImagePath(), checkpoint, this.gui);
switch (ret) {
case OK:
// no issue, exit the loop
@@ -1024,11 +1017,11 @@ import okhttp3.HttpUrl;
}
protected boolean shouldWaitBeforeRender() {
int concurrent_job = this.jobsToValidate.size();
int concurrentJob = this.jobsToValidate.size();
if (this.isValidatingJob) {
concurrent_job++;
concurrentJob++;
}
return (concurrent_job >= this.configuration.getMaxUploadingJob());
return (concurrentJob >= this.configuration.getMaxUploadingJob());
}
/****************
@@ -1036,7 +1029,7 @@ import okhttp3.HttpUrl;
* @int checkpoint - the checkpoint associated with the job (to add any additional log to the render output)
* @Job job - the job to be validated
*/
@AllArgsConstructor class QueuedJob {
@AllArgsConstructor private class QueuedJob {
final private int checkpoint;
final private Job job;
}