sheepit-shadow-nabber/src/main/java/com/sheepit/client/Client.java

/*
 * Copyright (C) 2010-2014 Laurent CLOUET
 * Author Laurent CLOUET <laurent.clouet@nopnop.net>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.sheepit.client;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Observable;
import java.util.Observer;
import java.util.Optional;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;

import com.sheepit.client.Error.ServerCode;
import com.sheepit.client.Error.Type;
import com.sheepit.client.datamodel.Chunk;
import com.sheepit.client.exception.SheepItException;
import com.sheepit.client.exception.SheepItExceptionNoRendererAvailable;
import com.sheepit.client.exception.SheepItExceptionNoRightToRender;
import com.sheepit.client.exception.SheepItExceptionNoSession;
import com.sheepit.client.exception.SheepItExceptionNoSpaceLeftOnDevice;
import com.sheepit.client.exception.SheepItExceptionPathInvalid;
import com.sheepit.client.exception.SheepItExceptionNoWritePermission;
import com.sheepit.client.exception.SheepItExceptionSessionDisabled;
import com.sheepit.client.exception.SheepItExceptionSessionDisabledDenoisingNotSupported;
import com.sheepit.client.exception.SheepItExceptionWithRequiredWait;
import com.sheepit.client.hardware.cpu.CPU;
import com.sheepit.client.os.OS;

import lombok.AllArgsConstructor;
import lombok.Data;
import okhttp3.HttpUrl;

@Data public class Client {
	public static final int MIN_JOB_ID = 20; //to distinguish between actual jobs and test frames
	private static final Locale LOCALE = Locale.ENGLISH;
	private DirectoryManager directoryManager;
	private Gui gui;
	private Server server;
	private Configuration configuration;
	private Log log;
	private Job renderingJob;
	private Job previousJob;
	private BlockingQueue<QueuedJob> jobsToValidate;
	private boolean isValidatingJob;
	private long startTime;
	private boolean sessionStarted;
	
	private boolean disableErrorSending;
	private boolean running;
	private boolean awaitingStop;
	private boolean suspended;
	private boolean shuttingdown;
	
	private int uploadQueueSize;
	private long uploadQueueVolume;
	private int noJobRetryIter;
	
	public Client(Gui gui, Configuration configuration, String url) {
		this.configuration = configuration;
		this.server = new Server(url, this.configuration, this);
		this.log = Log.getInstance(this.configuration);
		this.gui = gui;
		this.directoryManager = new DirectoryManager(this.configuration, this.log);
		this.renderingJob = null;
		this.previousJob = null;
		this.jobsToValidate = new ArrayBlockingQueue<>(5);
		this.isValidatingJob = false;
		
		this.disableErrorSending = false;
		this.running = false;
		this.awaitingStop = false;
		this.suspended = false;
		this.shuttingdown = false;
		
		this.uploadQueueSize = 0;
		this.uploadQueueVolume = 0;
		this.noJobRetryIter = 0;
		
		this.sessionStarted = false;
	}
	
	@Override public String toString() {
		return String.format("Client (configuration %s, server %s)", this.configuration, this.server);
	}
	
	public int run() {
		if (this.configuration.checkOSisSupported() == false) {
			this.gui.error(Error.humanString(Error.Type.OS_NOT_SUPPORTED));
			return -3;
		}
		
		if (this.configuration.checkCPUisSupported() == false) {
			this.gui.error(Error.humanString(Error.Type.CPU_NOT_SUPPORTED));
			return -4;
		}
		
		this.running = true;
		
		int step;
		try {
			step = this.log.newCheckPoint();
			this.gui.status("Starting");
			
			Error.Type ret;
			ret = this.server.getConfiguration();
			
			if (ret != Error.Type.OK) {
				this.gui.error(Error.humanString(ret));
				if (ret != Error.Type.AUTHENTICATION_FAILED) {
					Log.printCheckPoint(step);
				}
				return -1;
			}
			
			// If the user has selected to shutdown the computer at any specific time, set a scheduled task
			if (configuration.getShutdownTime() > 0) {
				new Timer().schedule(new TimerTask() {
					@Override public void run() {
						shuttingdown = true;
						log.debug("Initiating the computer's shutting down process");
						
						if ("wait".equals(configuration.getShutdownMode())) {
							// Soft stop. Complete current render (if any), finish uploading frames and then shutdown the computer
							askForStop();
						}
						else {
							// Soft stop. Interrupt the current render, abort pending uploads, close the client and shutdown the computer
							stop();
						}
					}
				}, this.configuration.getShutdownTime());
			}
			
			//send "error" log containing config and fs health check
			step = log.newCheckPoint();
			this.log.info("OS: " + OS.getOS().getVersion() + " " + System.getProperty("os.arch"));
			this.log.info(configuration.toString());
			for (String logline : configuration.filesystemHealthCheck()) {
				this.log.info(logline);
			}
			sendError(step, null, Type.OK);
			
			// Check integrity of all files in the working directories
			this.configuration.cleanWorkingDirectory();

			this.startTime = new Date().getTime();
			this.server.start(); // for staying alive
			
			// create a thread which will send the frame
			Runnable runnableSender = this::senderLoop;
			Thread threadSender = new Thread(runnableSender);
			threadSender.start();
			
			IncompatibleProcessChecker incompatibleProcessChecker = new IncompatibleProcessChecker(this);
			Timer incompatibleProcessCheckerTimer = new Timer();
			incompatibleProcessCheckerTimer.schedule(incompatibleProcessChecker, TimeUnit.MINUTES.toMillis(1), TimeUnit.MINUTES.toMillis(1));
			incompatibleProcessChecker.run(); // before the first request, check if it should be stopped
			
			do {
				while (this.running) {
					this.renderingJob = null;

					synchronized (this) {
						if (this.suspended) {
							if (incompatibleProcessChecker.isSuspendedDueToOtherProcess()) {
								this.gui.status("Client paused due to 'incompatible process' feature", true);
							}
							else {
								this.gui.status("Client paused", true);
								this.log.debug("Client paused");
							}
						}
						while (this.suspended && !this.shuttingdown) {
							wait();
						}
					}
					
					step = this.log.newCheckPoint();
					try {
						Calendar nextRequest = this.nextJobRequest();
						if (nextRequest != null) {
							// wait
							Date now = new Date();
							this.gui.status(String.format("Waiting until %tR before requesting job", nextRequest));
							long wait = nextRequest.getTimeInMillis() - now.getTime();
							if (wait < 0) {
								// it means the client has to wait until the next day
								wait += 24 * 3600 * 1000;
							}
							this.sleep(wait);
						}
						if (incompatibleProcessChecker.isRunningCompatibleProcess() == false) {
							this.gui.status("Requesting Job");
							this.renderingJob = this.server.requestJob();
						}
						else {
							this.gui.status("Wait until compatible process is stopped");
							this.sleep(30 * 1000);
						}
					}
					catch (SheepItExceptionNoRightToRender e) {
						this.gui.error("User does not have enough right to render project");
						return -2;
					}
					catch (SheepItExceptionSessionDisabled e) {
						this.gui.error(Error.humanString(Error.Type.SESSION_DISABLED));
						this.waitForever();
					}
					catch (SheepItExceptionSessionDisabledDenoisingNotSupported e) {
						this.gui.error(Error.humanString(Error.Type.DENOISING_NOT_SUPPORTED));
						this.waitForever();
					}
					catch (SheepItExceptionNoRendererAvailable e) {
						this.gui.error(Error.humanString(Error.Type.RENDERER_NOT_AVAILABLE));
						this.waitForever();
					}
					catch (SheepItExceptionNoSession e) {
						this.log.debug("User has no session and needs to re-authenticate");
						ret = this.server.getConfiguration();
						if (ret != Error.Type.OK) {
							this.renderingJob = null;
						}
						else {
							this.startTime = new Date().getTime(); // reset start session time because the server did it
							try {
								Calendar nextRequest = this.nextJobRequest();
								if (nextRequest != null) {
									// wait
									Date now = new Date();
									this.gui.status(String.format("Waiting until %tR before requesting job", nextRequest));
									long timeToSleep = nextRequest.getTimeInMillis() - now.getTime();
									this.activeSleep(timeToSleep);
								}

								// if we have broken the wait loop because a stop or shutdown signal, go back to the main loop to exit
								if (!this.running || this.shuttingdown) {
									continue;
								}

								this.gui.status("Requesting Job");
								this.renderingJob = this.server.requestJob();
							}
							catch (SheepItException e1) {
								this.renderingJob = null;
							}
						}
					}
					catch (SheepItExceptionWithRequiredWait e) {
						// exception example:
						// SheepItExceptionServerInMaintenance
						// SheepItExceptionServerOverloaded
						// SheepItServerDown
						// SheepItExceptionBadResponseFromServer
						
						int timeSleep = e.getWaitDuration();
						this.gui.status(String.format(e.getHumanText(), new Date(new Date().getTime() + timeSleep)));
						
						if (this.activeSleep(timeSleep) == false) {
							return -3;
						}
						this.log.removeCheckPoint(step);
						continue; // go back to ask job
					}
					catch (SheepItException e) {
						this.gui.error("Client::run exception requestJob (1) " + e.getMessage());
						StringWriter sw = new StringWriter();
						PrintWriter pw = new PrintWriter(sw);
						e.printStackTrace(pw);
						this.log.debug("Client::run exception " + e + " stacktrace: " + sw.toString());
						this.sendError(step);
						this.log.removeCheckPoint(step);
						continue;
					}
					
					if (this.renderingJob == null) { // no job
						int[] retrySchemeInMilliSeconds = { 300_000, 480_000, 720_000, 900_000, 1_200_000 };    // 5, 8, 12, 15 and 20 minutes
						
						int timeSleep = retrySchemeInMilliSeconds[(this.noJobRetryIter < retrySchemeInMilliSeconds.length) ?
								this.noJobRetryIter++ :
								(retrySchemeInMilliSeconds.length - 1)];
						this.gui.status(String.format("No job available. Will try again at %tR", new Date(new Date().getTime() + timeSleep)));
						if (this.activeSleep(timeSleep) == false) {
							return -3;
						}
						this.log.removeCheckPoint(step);
						continue; // go back to ask job
					}
					
					this.log.debug("Got work to do id: " + this.renderingJob.getId() + " frame: " + this.renderingJob.getFrameNumber());
					
					// As the server allocated a new job to this client, reset the no_job waiting algorithm
					this.noJobRetryIter = 0;
					
					ret = this.work(this.renderingJob);
					if (ret == Error.Type.NO_SPACE_LEFT_ON_DEVICE || ret == Error.Type.PATH_INVALID || ret == Error.Type.NO_WRITE_PERMISSION ) {
						Job frameToReset = this.renderingJob; // copy it because the sendError will take ~5min to execute
						this.renderingJob = null;
						this.gui.error(Error.humanString(ret));
						this.sendError(step, frameToReset, ret);
						this.log.removeCheckPoint(step);
						return -50;
					}
					
					if (ret != Error.Type.OK) {
						Job currentJob = this.renderingJob; // copy it because the sendError will take ~5min to execute
						this.renderingJob = null;
						this.gui.error(Error.humanString(ret));
						this.sendError(step, currentJob, ret);
						this.log.removeCheckPoint(step);
						
						// Initial test frames always have the Job ID below 20. If we have any error while trying to render the initial frame just
						// halt the execution
						if (Integer.parseInt(currentJob.getId()) < MIN_JOB_ID) {
							// Add the proper explanation to the existing error message and keep the client waiting forever to ensure the user sees the error
							this.gui.error(Error.humanString(ret) + " The error happened during the test frame render. Restart the client and try again.");
							this.waitForever();
							break;	// if the shutdown signal is triggered then exit the while (this.running) loop to initiate the shutdown process
						}
						
						continue;
					}
					
					if (this.renderingJob.isSynchronousUpload()) { // power or compute_method job, need to upload right away
						this.gui.status(String.format("Uploading frame (%.2fMB)", (this.renderingJob.getOutputImageSize() / 1024.0 / 1024.0)));
						
						ret = confirmJob(this.renderingJob, step);
						if (ret != Error.Type.OK) {
							gui.error("Client::run problem with confirmJob (returned " + ret + ")");
							sendError(step, this.renderingJob, Error.Type.VALIDATION_FAILED);
						}
						this.renderingJob = null;
					}
					else {
						this.gui.status(String.format("Queuing frame for upload (%.2fMB)", (this.renderingJob.getOutputImageSize() / 1024.0 / 1024.0)));
						
						this.jobsToValidate.add(new QueuedJob(step, this.renderingJob));
						
						this.uploadQueueSize++;
						this.uploadQueueVolume += this.renderingJob.getOutputImageSize();
						this.gui.displayUploadQueueStats(uploadQueueSize, uploadQueueVolume);
						
						this.renderingJob = null;
					}
					
					if (this.shouldWaitBeforeRender()) {
						this.gui.status("Sending frames. Please wait");
						
						while (this.shouldWaitBeforeRender()) {
							this.sleep(4000);
						}
					}
					this.log.removeCheckPoint(step);
				}
				
				// If we reach this point is bc the main loop (the one that controls all the workflow) has exited
				// due to user requesting to exit the App and we are just waiting for the upload queue to empty
				// If the user cancels the exit, then this.running will be true and the main loop will take
				// control again
				this.gui.status("Uploading rendered frames before exiting. Please wait");
				this.sleep(2300);
				
				// This loop will remain valid until all the background uploads have
				// finished (unless the stop() method has been triggered)
			}
			while (this.uploadQueueSize > 0);
		}
		catch (Exception e1) {
			// no exception should be raised in the actual launcher (applet or standalone)
			StringWriter sw = new StringWriter();
			PrintWriter pw = new PrintWriter(sw);
			e1.printStackTrace(pw);
			this.log.debug("Client::run exception(D) " + e1 + " stacktrace: " + sw.toString());
			return -99; // the this.stop will be done after the return of this.run()
		}
		
		if (this.shuttingdown) {
			// Shutdown the computer using the appropriate command for the host OS
			this.log.debug("Shutting down the computer in 1 minute");
			
			OS.getOS().shutdownComputer(1);
		}
		
		this.gui.stop();
		return 0;
	}
	
	public synchronized int stop() {
		this.running = false;
		this.disableErrorSending = true;
		
		if (this.renderingJob != null) {
			this.gui.status("Stopping");
			
			if (this.renderingJob.getProcessRender().getProcess() != null) {
				this.renderingJob.setAskForRendererKill(true);
				this.renderingJob.getProcessRender().kill();
			}
		}

		this.configuration.removeWorkingDirectory();
		
		if (this.server == null) {
			return 0;
		}
		
		if (this.server.getPage("logout").isEmpty() == false) {
			this.gui.status("Disconnecting from SheepIt server");
			
			try {
				this.server.HTTPRequest(this.server.getPage("logout"));
			}
			catch (IOException ignored) {
				// nothing to do: if the logout failed that's ok
			}
		}
		this.server.interrupt();
		try {
			this.server.join();
		}
		catch (InterruptedException e) {
		}
		
		this.server = null;
		
		return 0;
	}
	
	public void suspend() {
		suspended = true;
		this.gui.status("Client will pause when the current job finishes", true);
	}
	
	public synchronized void resume() {
		suspended = false;
		notify();
	}
	
	public void waitForever() {
		// the client is most likely dead.
		// instead of exiting, wait forever to display an error message on the UI
		
		while (shuttingdown == false) {
			try {
				Thread.sleep(1000);
			}
			catch (InterruptedException ignored) {
			}
		}
	}
	
	/**
	 * Sleep but stop if the client is not running or shutting down
	 * @param wait in ms
	 */
	public boolean activeSleep(long wait) {
		try {
			long timeSlept = 0;
			while (timeSlept < wait && this.running && this.shuttingdown == false) {
				Thread.sleep(1000);
				timeSlept += 1000;
			}
			return true;
		}
		catch (InterruptedException ignored) {
			return false;
		}
	}
	
	/**
	 * @param wait in ms
	 */
	public boolean sleep(long wait) {
		try {
			int timeSlept = 0;
			while (timeSlept < wait) {
				Thread.sleep(1000);
				timeSlept += 1000;
			}
			return true;
		}
		catch (InterruptedException ignored) {
			return false;
		}
	}
	
	public void askForStop() {
		this.log.debug("Client::askForStop");
		this.running = false;
		this.awaitingStop = true;
	}
	
	public void cancelStop() {
		this.log.debug("Client::cancelStop");
		this.running = true;
		this.awaitingStop = false;
	}
	
	public int senderLoop() {
		int step = -1;
		Error.Type ret = null;
		while (true) {
			QueuedJob queuedJob = null;
			try {
				queuedJob = jobsToValidate.take();
				step = queuedJob.checkpoint;	// retrieve the checkpoint attached to the job
				
				this.log.debug(step, "will validate " + queuedJob.job);
				
				ret = confirmJob(queuedJob.job, step);
				if (ret != Error.Type.OK) {
					this.gui.error(Error.humanString(ret));
					this.log.debug(step, "Client::senderLoop confirm failed, ret: " + ret);
				}
			}
			catch (InterruptedException e) {
				this.log.error(step, "Client::senderLoop Exception " + e.getMessage());
			}
			finally {
				if (ret != Error.Type.OK) {
					if (queuedJob.job != null) {
						sendError(step, queuedJob.job, ret);
					}
					else {
						sendError(step);
					}
				}
				
				// Remove the checkpoint information
				log.removeCheckPoint(step);
				
				this.uploadQueueSize--;
				if (queuedJob.job != null) {
					this.uploadQueueVolume -= queuedJob.job.getOutputImageSize();
				}
				
				this.gui.displayUploadQueueStats(this.uploadQueueSize, this.uploadQueueVolume);
			}
		}
	}
	
	protected void sendError(int step) {
		this.sendError(step, null, null);
	}
	
	protected void sendError(int step, Job jobToReset, Error.Type error) {
		if (this.disableErrorSending) {
			this.log.debug("Error sending is disabled, do not send log");
			return;
		}
		
		this.log.debug("Sending error to server (type: " + error + ")");
		try {
			File tempFile = File.createTempFile("farm_", ".txt");
			tempFile.createNewFile();
			tempFile.deleteOnExit();
			FileOutputStream writer = new FileOutputStream(tempFile);
			
			// Create a header with the information summarised for easier admin error analysis
			Configuration conf = this.configuration;
			CPU cpu = OS.getOS().getCPU();
			
			StringBuilder logHeader = new StringBuilder()
				.append("====================================================================================================\n")
				.append(String.format("%s  /  %s  /  %s  /  SheepIt v%s\n", conf.getLogin(), conf.getHostname(), OS.getOS().name(), Configuration.jarVersion))
				.append(String.format("%s  x%d  %.1f GB RAM\n", cpu.getName(), conf.getNbCores(), conf.getMaxAllowedMemory() / 1024.0 / 1024.0));
			
			if (conf.getComputeMethod() == Configuration.ComputeType.GPU || conf.getComputeMethod() == Configuration.ComputeType.CPU_GPU) {
				logHeader.append(String.format("%s   %s   %.1f GB VRAM\n", conf.getGPUDevice().getId(), conf.getGPUDevice().getModel(),
					conf.getGPUDevice().getMemory() / 1024.0 / 1024.0 / 1024.0));
			}
			
			logHeader.append("====================================================================================================\n");
			if (jobToReset != null) {
				logHeader.append(String.format("Project ::: %s\n", jobToReset.getName()))
					.append(String.format("Project id: %s  frame: %s\n", jobToReset.getId(), jobToReset.getFrameNumber())).append(String.format("blender ::: %s\n\n", jobToReset.getBlenderLongVersion())).append(String.format("ERROR Type :: %s\n", error));
			}
			else {
				logHeader.append("Project ::: No project allocated.\n")
						.append(String.format("ERROR Type :: %s\n", (error != null ? error : "N/A")));
			}
			logHeader.append("====================================================================================================\n\n");
			
			// Insert the info at the beginning of the error log
			writer.write(logHeader.toString().getBytes());
			
			Optional<ArrayList<String>> logs = this.log.getForCheckPoint(step);
			if (logs.isPresent()) {
				for (String line : logs.get()) {
					writer.write(line.getBytes());
					writer.write('\n');
				}
			}
			
			writer.close();
			HttpUrl.Builder remoteURL = HttpUrl.parse(this.server.getPage("error")).newBuilder();
			remoteURL.addQueryParameter("type", error == null ? "" : Integer.toString(error.getValue()));
			if (jobToReset != null) {
				remoteURL.addQueryParameter("frame", jobToReset.getFrameNumber());
				remoteURL.addQueryParameter("job", jobToReset.getId());
				remoteURL.addQueryParameter("render_time", Integer.toString(jobToReset.getProcessRender().getRenderDuration()));
				remoteURL.addQueryParameter("memoryused", Long.toString(jobToReset.getProcessRender().getPeakMemoryUsed()));
			}
			this.server.HTTPSendFile(remoteURL.build().toString(), tempFile.getAbsolutePath(), step, this.gui);
			tempFile.delete();
		}
		catch (Exception e) {
			StringWriter sw = new StringWriter();
			PrintWriter pw = new PrintWriter(sw);
			e.printStackTrace(pw);
			this.log.debug("Client::sendError Exception " + e + " stacktrace: " + sw.toString());
			// no exception should be raised to actual launcher (applet or standalone)
		}
	}
	
	/**
	 * @return the date of the next request, or null if there is not delay (null <=> now)
	 */
	public Calendar nextJobRequest() {
		if (this.configuration.getRequestTime() == null) {
			return null;
		}
		else {
			Calendar next = null;
			
			List<Calendar> dates = new ArrayList<>();
			
			Calendar now = Calendar.getInstance();
			for (Pair<Calendar, Calendar> interval : this.configuration.getRequestTime()) {
				Calendar start = (Calendar) now.clone();
				Calendar end = (Calendar) now.clone();
				start.set(Calendar.SECOND, 00);
				start.set(Calendar.MINUTE, interval.first.get(Calendar.MINUTE));
				start.set(Calendar.HOUR_OF_DAY, interval.first.get(Calendar.HOUR_OF_DAY));
				
				end.set(Calendar.SECOND, 59);
				end.set(Calendar.MINUTE, interval.second.get(Calendar.MINUTE));
				end.set(Calendar.HOUR_OF_DAY, interval.second.get(Calendar.HOUR_OF_DAY));
				
				if (start.before(now) && now.before(end)) {
					return null;
				}
				
				Calendar startTomorow = (Calendar) start.clone();
				startTomorow.add(Calendar.DATE, 1);
				dates.add(start);
				dates.add(startTomorow);
			}
			
			for (Calendar cal : dates) {
				if (cal.after(now) && (next == null || (cal.getTimeInMillis() - now.getTimeInMillis() < next.getTimeInMillis() - now.getTimeInMillis()))) {
					next = cal;
				}
			}
			
			return next;
		}
	}
	
	public Error.Type work(final Job ajob) {
		Error.Type downloadRet;
		
		gui.setRenderingProjectName(ajob.getName());
		
		try {
			downloadRet = this.downloadExecutable(ajob);
			if (downloadRet != Error.Type.OK) {
				gui.setRenderingProjectName("");
				for (String logline : configuration.filesystemHealthCheck()) {
					log.debug(logline);
				}
				this.log.error("Client::work problem with downloadExecutable (ret " + downloadRet + ")");
				return downloadRet;
			}
			
			downloadRet = this.downloadSceneFile(ajob);
			if (downloadRet != Error.Type.OK) {
				gui.setRenderingProjectName("");
				for (String logline : configuration.filesystemHealthCheck()) {
					log.debug(logline);
				}
				this.log.error("Client::work problem with downloadSceneFile (ret " + downloadRet + ")");
				return downloadRet;
			}
			
			int ret = this.prepareWorkingDirectory(ajob); // decompress renderer and scene archives
			if (ret != 0) {
				gui.setRenderingProjectName("");
				for (String logline : configuration.filesystemHealthCheck()) {
					log.debug(logline);
				}
				this.log.error("Client::work problem with this.prepareWorkingDirectory (ret " + ret + ")");
				return Error.Type.CAN_NOT_CREATE_DIRECTORY;
			}
		}
		catch (SheepItException e) {
			gui.setRenderingProjectName("");
			for (String logline : configuration.filesystemHealthCheck()) {
				log.debug(logline);
			}
			if (e instanceof SheepItExceptionNoSpaceLeftOnDevice) {
				return Error.Type.NO_SPACE_LEFT_ON_DEVICE;
			}
			else if (e instanceof SheepItExceptionPathInvalid) {
				return Error.Type.PATH_INVALID;
			}
			else if (e instanceof SheepItExceptionNoWritePermission) {
				return Error.Type.NO_WRITE_PERMISSION;
			}
			else {
				return Error.Type.UNKNOWN;
			}
		}
		
		final File sceneFile = new File(ajob.getScenePath());
		File rendererFile = new File(ajob.getRendererPath());
		
		if (sceneFile.exists() == false) {
			gui.setRenderingProjectName("");
			for (String logline : configuration.filesystemHealthCheck()) {
				log.debug(logline);
			}
			this.log.error("Client::work job preparation failed (scene file '" + sceneFile.getAbsolutePath()
					+ "' does not exist), cleaning directory in hope to recover");
			this.configuration.cleanWorkingDirectory();
			return Error.Type.MISSING_SCENE;
		}
		
		if (rendererFile.exists() == false) {
			gui.setRenderingProjectName("");
			for (String logline : configuration.filesystemHealthCheck()) {
				log.debug(logline);
			}
			this.log.error("Client::work job preparation failed (renderer file '" + rendererFile.getAbsolutePath()
					+ "' does not exist), cleaning directory in hope to recover");
			this.configuration.cleanWorkingDirectory();
			return Error.Type.MISSING_RENDERER;
		}
		
		Observer removeSceneDirectoryOnceRenderHasStartedObserver = new Observer() {
			@Override public void update(Observable observable, Object o) {
				// only remove the .blend since it's most important data
				// and it's the only file we are sure will not be needed anymore
				sceneFile.delete();
			}
		};
		
		Error.Type err = ajob.render(removeSceneDirectoryOnceRenderHasStartedObserver);
		gui.setRenderingProjectName("");
		gui.setRemainingTime("");
		gui.setRenderingTime("");
		gui.setComputeMethod("");
		
		removeSceneDirectory(ajob);
		
		if (err != Error.Type.OK) {
			this.log.error("Client::work problem with runRenderer (ret " + err + ")");
		}
		return err;
	}
	
	protected Error.Type downloadSceneFile(Job ajob) throws SheepItException {
		int total = ajob.getArchiveChunks().size();
		int threads = Math.max(1, Math.min(total, 12)); // at least one thread, to avoid IllegalArgumentException if total = 0
		
		ExecutorService executor = Executors.newFixedThreadPool(threads);
		ArrayList<Callable<Error.Type>> tasks = new ArrayList<>();
		
		this.gui.getDownloadProgress().reset("Downloading project");
		
		for (int i = 0; i < total; i++) {
			Chunk chunk = ajob.getArchiveChunks().get(i);
			
			Callable<Type> downloadTask = () -> {
				DownloadManager downloadManager  = new DownloadManager(
					this.server,
					this.gui,
					this.log,
					this.directoryManager.getActualStoragePathFor(chunk),
					chunk.getMd5(),
					String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
				);
				return downloadManager.download();
			};
			
			tasks.add(downloadTask);
		}
		
		try {
			var results = executor.invokeAll(tasks);
			
			for (var result : results) {
				if (result.get(35, TimeUnit.MINUTES) != Type.OK) {
					executor.shutdown();
					return result.get(35, TimeUnit.MINUTES);
				}
			}
		}
		catch (ExecutionException | InterruptedException | TimeoutException e) {
			
			executor.shutdown();
			return Type.DOWNLOAD_FILE;
		}

		
		executor.shutdown();
		return Type.OK;
	}
	
	protected Error.Type downloadExecutable(Job ajob) throws SheepItException {
		this.gui.getDownloadProgress().reset("Downloading Blender");
		return (new DownloadManager(
			this.server,
			this.gui,
			this.log,
			this.directoryManager.getActualStorageBinaryPathFor(ajob),
			ajob.getRendererMD5(),
			String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
		)).download();
	}
	
	protected void removeSceneDirectory(Job ajob) {
		Utils.delete(new File(ajob.getSceneDirectory()));
	}
	
	protected int prepareWorkingDirectory(Job ajob) {
		int ret;
		
		String rendererArchive = this.directoryManager.getCacheBinaryPathFor(ajob);
		String rendererPath = ajob.getRendererDirectory();
		File rendererPathFile = new File(rendererPath);
		
		// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
		if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
			this.gui.status("Copying renderer from shared downloads directory");
			this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
			if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
				log.error("Error while copying " + rendererArchive + " from shared downloads directory to working dir");
			}
		}
		
		if (!rendererPathFile.exists()) {
			// we create the directory
			rendererPathFile.mkdir();
			
			this.gui.status("Extracting renderer");
			this.log.debug("Client::prepareWorkingDirectory Extracting renderer " + rendererArchive + " into " + rendererPath);
			
			// unzip the archive
			ret = Utils.unzipFileIntoDirectory(rendererArchive, rendererPath, null, log);
			if (ret != 0) {
				this.log.error(
						"Client::prepareWorkingDirectory, error(1) with Utils.unzipFileIntoDirectory(" + rendererArchive + ", " + rendererPath + ") returned "
								+ ret);
				this.gui.error(String.format("Unable to extract the renderer (error %d)", ret));
				return -1;
			}
			
			try {
				File f = new File(ajob.getRendererPath());
				f.setExecutable(true);
			}
			catch (SecurityException e) {
			}
		}
		
		String scenePath = ajob.getSceneDirectory();
		File scenePathFile = new File(scenePath);
		
		// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
		for (Chunk chunk: ajob.getArchiveChunks()) {
			if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
				this.gui.status("Copying chunk from common directory");
				if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
					this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
				}
			}
			
		}
		
		/// download the chunks
		
		if (!scenePathFile.exists()) {
			// we create the directory
			scenePathFile.mkdir();
			
			this.gui.status("Extracting project");
			this.log.debug("Client::prepareWorkingDirectory Extracting project into " + scenePath);
			// unzip the archive
			
			Instant startUnzip = Instant.now();
			ret = Utils.unzipChunksIntoDirectory(
				ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
				scenePath,
				ajob.getPassword(),
				log);
			
			Instant stopUnzip = Instant.now();
			Duration unzipDuration = Duration.between(startUnzip, stopUnzip);
			log.debug("Unzipping " + ajob.getArchiveChunks().size() + " chunks of \"" + ajob.getName() + "\" took " + unzipDuration.toSeconds() + "s");
			
			if (ret != 0) {
				this.log.error("Client::prepareWorkingDirectory, error(2) with Utils.unzipChunksIntoDirectory returned " + ret);
				this.gui.error(String.format("Unable to extract the scene (error %d)", ret));
				return -2;
			}
		}
		
		return 0;
	}

	protected Error.Type confirmJob(Job ajob, int checkpoint) {
		String urlReal = String.format(LOCALE, "%s&rendertime=%d&preptime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(), ajob.getProcessRender().getScenePrepDuration(),
				ajob.getProcessRender().getPeakMemoryUsed());
		if (ajob.getSpeedSamplesRendered() > 0.0) {
			urlReal +=  String.format(LOCALE, "&speedsamples=%s", ajob.getSpeedSamplesRendered());
		}
		this.log.debug(checkpoint, "Client::confirmeJob url " + urlReal);
		this.log.debug(checkpoint, "path frame " + ajob.getOutputImagePath());
		
		this.isValidatingJob = true;
		int maxTries = 3;
		int timeToSleep = 22_000;
		ServerCode ret;
		Type confirmJobReturnCode = Error.Type.OK;
		retryLoop:
		for (int nbTry = 0; nbTry < maxTries; nbTry++) {
			if (nbTry >= 1) {
				// sleep before retrying
				this.log.debug(checkpoint, "Sleep for " + timeToSleep / 1000 + "s before trying to re-upload the frame, previous error: "+ confirmJobReturnCode);
				try {
					Thread.sleep(timeToSleep);
				}
				catch (InterruptedException e) {
					confirmJobReturnCode = Error.Type.UNKNOWN;
				}
				timeToSleep *= 2;    // exponential backoff
			}

			ret = this.server.HTTPSendFile(urlReal, ajob.getOutputImagePath(), checkpoint, this.gui);
			switch (ret) {
				case OK:
					// no issue, exit the loop
					break retryLoop;
				
				case JOB_VALIDATION_ERROR_SESSION_DISABLED:
				case JOB_VALIDATION_ERROR_BROKEN_MACHINE:
					confirmJobReturnCode = Error.Type.SESSION_DISABLED;
					break retryLoop;
				
				case JOB_VALIDATION_ERROR_IMAGE_WRONG_DIMENSION:
					confirmJobReturnCode = Error.Type.IMAGE_WRONG_DIMENSION;
					break retryLoop;
					
				case JOB_VALIDATION_ERROR_MISSING_PARAMETER:
					// no point to retry the request
					confirmJobReturnCode = Error.Type.UNKNOWN;
					break retryLoop;
					
				case JOB_VALIDATION_IMAGE_TOO_LARGE:
					// the client cannot recover from this error (it's server side config) so exit the retry loop
					confirmJobReturnCode = Type.IMAGE_TOO_LARGE;
					break retryLoop;
					
				case SERVER_CONNECTION_FAILED:
					confirmJobReturnCode = Type.NETWORK_ISSUE;
					break;
				
				case ERROR_BAD_RESPONSE:
					// set the error and retry on next loop
					confirmJobReturnCode = Type.ERROR_BAD_UPLOAD_RESPONSE;
					break;
					
				default:
					// do nothing, try to do a request on the next loop
					break;
			}
		}
		
		this.isValidatingJob = false;
		this.previousJob = ajob;
		
		//count frames if they are not test frames and got validated correctly
		if (confirmJobReturnCode == Error.Type.OK && Integer.parseInt(ajob.getId()) >= MIN_JOB_ID) {
			gui.AddFrameRendered();
		}
		
		// we can remove the frame file
		Utils.deleteFile(new File(ajob.getOutputImagePath()));
		ajob.setOutputImagePath(null);
		
		if (ajob.getPreviewImagePath() != null) {
			Utils.deleteFile(new File(ajob.getPreviewImagePath()));
			ajob.setPreviewImagePath(null);
		}
		
		return confirmJobReturnCode;
	}
	
	protected boolean shouldWaitBeforeRender() {
		int concurrentJob = this.jobsToValidate.size();
		if (this.isValidatingJob) {
			concurrentJob++;
		}
		return (concurrentJob >= this.configuration.getMaxUploadingJob());
	}
	
	/****************
	 * Inner class that will hold the queued jobs. The constructor accepts two parameters:
	 * @int checkpoint - the checkpoint associated with the job (to add any additional log to the render output)
	 * @Job job - the job to be validated
	 */
	@AllArgsConstructor private class QueuedJob {
		final private int checkpoint;
		final private Job job;
	}
}