sheepit-shadow-nabber/src/main/java/com/sheepit/client/Job.java

/*
 * Copyright (C) 2010-2014 Laurent CLOUET
 * Author Laurent CLOUET <laurent.clouet@nopnop.net>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.sheepit.client;

import com.sheepit.client.Configuration.ComputeType;
import com.sheepit.client.Error.Type;
import com.sheepit.client.os.OS;
import lombok.Data;
import lombok.Getter;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Observable;
import java.util.Observer;
import java.util.Optional;
import java.util.TimeZone;
import java.util.Timer;
import java.util.TimerTask;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Data public class Job {
	public static final String UPDATE_METHOD_BY_REMAINING_TIME = "remainingtime";
	public static final String UPDATE_METHOD_BY_TILE = "by_tile";
	public static final String POST_LOAD_NOTIFICATION = "POST_LOAD_SCRIPT_loaded";
	
	public static final int SHOW_BASE_ICON = -1;
	
	private String frameNumber;
	private DownloadItem projectDownload;
	private DownloadItem rendererDownload;
	private String id;
	private String outputImagePath;
	
	private String previewImagePath;
	private long outputImageSize;
	private String path; // path inside of the archive
	private String rendererCommand;
	private String validationUrl;
	private String script;
	private boolean useGPU;
	private String name;
	private char[] password;
	private String updateRenderingStatusMethod;
	private String blenderShortVersion;
	private String blenderLongVersion;
	private boolean synchronousUpload;
	private float speedSamplesRendered; // only used for powerdetection project
	private RenderProcess render;
	private boolean askForRendererKill;
	private boolean userBlockJob;
	private boolean serverBlockJob;
	private boolean incompatibleProcessKillJob;
	private Gui gui;
	private Configuration configuration;
	private Log log;
	
	public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
			String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
			String update_method_) {
		configuration = config_;
		id = id_;
		frameNumber = frame_;
		path = path_;
		useGPU = use_gpu;
		rendererCommand = command_;
		validationUrl = validationUrl_;
		projectDownload = projectDownload_;
		rendererDownload = rendererDownload_;
		name = name_;
		password = password_.clone();
		synchronousUpload = synchronous_upload_;
		gui = gui_;
		outputImagePath = null;
		outputImageSize = 0;
		script = script_;
		updateRenderingStatusMethod = update_method_;
		askForRendererKill = false;
		userBlockJob = false;
		serverBlockJob = false;
		incompatibleProcessKillJob = false;
		log = log_;
		render = new RenderProcess(log_);
		blenderShortVersion = null;
		blenderLongVersion = null;
		speedSamplesRendered = 0;
	}
	
	public void block() {
		setAskForRendererKill(true);
		setUserBlockJob(true);
		RenderProcess process = getProcessRender();
		if (process != null) {
			process.kill();
		}
	}
	
	public void incompatibleProcessBlock() {
		setAskForRendererKill(true);
		setIncompatibleProcessKillJob(true);
		RenderProcess process = getProcessRender();
		if (process != null) {
			process.kill();
		}
	}
	
	public RenderProcess getProcessRender() {
		return render;
	}
	
	@Override
	public String toString() {
		return String
				.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
						frameNumber, projectDownload.getMD5(), rendererDownload.getMD5(), id, outputImagePath, path, useGPU, name, updateRenderingStatusMethod, render);
	}
	
	public String getPrefixOutputImage() {
		return id + "_";
	}
	
	public String getRendererDirectory() {
		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererDownload.getMD5();
	}
	
	public String getRendererPath() {
		return getRendererDirectory() + File.separator + OS.getOS().getRenderBinaryPath();
	}
	
	public String getSceneDirectory() {
		return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id;
	}
	
	public String getScenePath() {
		return getSceneDirectory() + File.separator + this.path;
	}
	
	public Error.Type render(Observer renderStarted) {
		gui.status("Rendering");
		RenderProcess process = getProcessRender();
		Timer timerOfMaxRenderTime = null;
		String core_script;
		// When sending Ctrl+C to the terminal it also get's sent to all subprocesses e.g. also the render process.
		// The java program handles Ctrl+C but the renderer quits on Ctrl+C.
		// This script causes the renderer to ignore Ctrl+C.
		String ignore_signal_script = "import signal\n" + "def hndl(signum, frame):\n" + "    pass\n" + "signal.signal(signal.SIGINT, hndl)\n";
		if (isUseGPU() && configuration.getGPUDevice() != null && configuration.getComputeMethod() != ComputeType.CPU) {
			core_script = "sheepit_set_compute_device(\"" + configuration.getGPUDevice().getType() + "\", \"GPU\", \"" + configuration.getGPUDevice().getId()
					+ "\")\n";
			gui.setComputeMethod("GPU");
		}
		else {
			// Otherwise (CPU), fix the tile size to 32x32px
			core_script = "sheepit_set_compute_device(\"NONE\", \"CPU\", \"CPU\")\n";
			gui.setComputeMethod("CPU");
		}
		
		core_script += ignore_signal_script;
		File disableViewportScript = null;
		File script_file = null;
		String[] command1 = getRendererCommand().split(" ");
		int size_command = command1.length + 2; // + 2 for script
		
		if (configuration.getNbCores() > 0) { // user has specified something
			size_command += 2;
		}
		
		List<String> command = new ArrayList<>(size_command);
		
		Map<String, String> new_env = new HashMap<>();
		
		//make sure the system doesn´t interfere with the blender runtime, and that blender doesn´t attempt to load external libraries/scripts.
		new_env.put("BLENDER_USER_RESOURCES", "");
		new_env.put("BLENDER_USER_CONFIG", "");
		new_env.put("BLENDER_USER_EXTENSIONS", "");
		new_env.put("BLENDER_USER_SCRIPTS", "");
		new_env.put("BLENDER_USER_DATAFILES", "");

		new_env.put("BLENDER_SYSTEM_RESOURCES", "");
		new_env.put("BLENDER_SYSTEM_SCRIPTS", "");
		new_env.put("BLENDER_SYSTEM_DATAFILES", "");
		new_env.put("BLENDER_SYSTEM_PYTHON", "");

		new_env.put("OCIO", ""); //prevent blender from loading a non-standard color configuration
		new_env.put("TEMP", configuration.getWorkingDirectory().getAbsolutePath().replace("\\", "\\\\"));
		new_env.put("TMP", configuration.getWorkingDirectory().getAbsolutePath().replace("\\", "\\\\"));
		
		new_env.put("CORES", Integer.toString(configuration.getNbCores()));
		new_env.put("PRIORITY", Integer.toString(configuration.getPriority()));

		// Add own lib folder first, because Steam or other environments may set an LD_LIBRARY_PATH that has priority over the runpath in the Blender excutable,
		// but contains incompatible libraries.
		String currentLDLibraryPath = Optional.ofNullable(System.getenv("LD_LIBRARY_PATH")).orElse("");
		new_env.put("LD_LIBRARY_PATH", getRendererDirectory() + "/lib" + ":" + currentLDLibraryPath);
		
		for (String arg : command1) {
			switch (arg) {
				case ".c":
					command.add("-P");
					
					try {
						disableViewportScript = File.createTempFile("pre_load_script_", ".py", configuration.getWorkingDirectory());
						File file = new File(disableViewportScript.getAbsolutePath());
						FileWriter fwriter;
						fwriter = new FileWriter(file);
						
						PrintWriter out = new PrintWriter(fwriter);
						out.write("import bpy");
						out.write("\n");
						out.write("import sys");
						out.write("\n");
						out.write("from bpy.app.handlers import persistent");
						out.write("\n");
						out.write("@persistent");
						out.write("\n");
						out.write("def hide_stuff(hide_dummy):");
						out.write("\n");
						out.write("	print('PRE_LOAD_SCRIPT_hide_viewport')");
						out.write("\n");
						out.write("	#Hide collections in the viewport");
						out.write("\n");
						out.write("	for col in bpy.data.collections:");
						out.write("\n");
						out.write("		col.hide_viewport = True");
						out.write("\n");
						out.write("	for obj in bpy.data.objects:");
						out.write("\n");
						out.write("		#Hide objects in the viewport");
						out.write("\n");
						out.write("		#obj.hide_viewport = True");
						out.write("\n");
						out.write("		#Hide modifier in the viewport");
						out.write("\n");
						out.write("		for mod in obj.modifiers:");
						out.write("\n");
						out.write("			mod.show_viewport = False");
						out.write("\n");
						out.write("	sys.stdout.flush()");
						out.write("\n");
						out.write("bpy.app.handlers.version_update.append(hide_stuff)");
						out.write("\n");
						out.close();
						
						command.add(disableViewportScript.getAbsolutePath());
					}
					catch (IOException e) {
						StringWriter sw = new StringWriter();
						e.printStackTrace(new PrintWriter(sw));
						for (String logline : configuration.filesystemHealthCheck()) {
							log.debug(logline);
						}
						log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
						return Error.Type.UNKNOWN;
					}
					
					command.add(getScenePath());
					command.add("-P");
					
					try {
						script_file = File.createTempFile("post_load_script_", ".py", configuration.getWorkingDirectory());
						File file = new File(script_file.getAbsolutePath());
						FileWriter txt;
						txt = new FileWriter(file);
						
						PrintWriter out = new PrintWriter(txt);
						out.write(getScript());
						out.write("\n");
						out.write("import sys");
						out.write("\n");
						out.write("print('" + POST_LOAD_NOTIFICATION + "')");
						out.write("\n");
						out.write("sys.stdout.flush()");
						out.write("\n");
						out.write(core_script); // GPU part
						out.write("\n"); // GPU part
						out.close();
						
						command.add(script_file.getAbsolutePath());
					}
					catch (IOException e) {
						StringWriter sw = new StringWriter();
						e.printStackTrace(new PrintWriter(sw));
						for (String logline : configuration.filesystemHealthCheck()) {
							log.debug(logline);
						}
						log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
						return Error.Type.UNKNOWN;
					}
					script_file.deleteOnExit();
					break;
				case ".e":
					command.add(getRendererPath());
					// the number of cores has to be put after the binary and before the scene arg
					if (configuration.getNbCores() > 0) {
						command.add("-t");
						command.add(Integer.toString(configuration.getNbCores()));
					}
					break;
				case ".o":
					command.add(configuration.getWorkingDirectory().getAbsolutePath() + File.separator + getPrefixOutputImage());
					break;
				case ".f":
					command.add(getFrameNumber());
					break;
				default:
					command.add(arg);
					break;
			}
		}
		
		Timer memoryCheck = new Timer();
		try {
			renderStartedObservable event = new renderStartedObservable(renderStarted);
			String line;
			log.debug(command.toString());
			OS os = OS.getOS();
			process.setCoresUsed(configuration.getNbCores());
			process.start();
			getProcessRender().setProcess(os.exec(command, new_env));
			getProcessRender().setOsProcess(OS.operatingSystem.getProcess((int) getProcessRender().getProcess().pid()));
			BufferedReader input = new BufferedReader(new InputStreamReader(getProcessRender().getProcess().getInputStream()));
			memoryCheck.scheduleAtFixedRate(new TimerTask() {
				@Override public void run() {
					updateProcess();
				}
			}, 0L, 200L);
			
			// Make initial test/power frames ignore the maximum render time in user configuration. Initial test frames have Job IDs below 20
			// so we just activate the user defined timeout when the scene is not one of the initial ones.
			if (configuration.getMaxRenderTime() > 0 && Integer.parseInt(this.getId()) >= 20) {
				timerOfMaxRenderTime = new Timer();
				timerOfMaxRenderTime.schedule(new TimerTask() {
					@Override public void run() {
						RenderProcess process = getProcessRender();
						if (process != null) {
							long duration = (new Date().getTime() - process.getStartTime()) / 1000; // in seconds
							if (configuration.getMaxRenderTime() > 0 && duration > configuration.getMaxRenderTime()) {
								setAskForRendererKill(true);
								log.debug("Killing render - exceeding allowed process duration");
								process.kill();
							}
						}
					}
				}, configuration.getMaxRenderTime() * 1000 + 2000); // +2s to be sure the delay is over
			}
			
			
			log.debug("renderer output");
			try {
				int progress = -1;
				
				Pattern progressPattern = Pattern.compile(" (Rendered|Path Tracing Tile|Rendering|Sample) (\\d+)\\s?\\/\\s?(\\d+)( Tiles| samples|,)*");
				Pattern beginScenePrepPattern = Pattern.compile(POST_LOAD_NOTIFICATION);
				Pattern beginPostProcessingPattern = Pattern.compile("^Fra:\\d* \\w*(.)* \\| (Compositing|Denoising)");
				Pattern savingPattern = Pattern.compile("Time: \\d\\d:\\d\\d.\\d\\d \\(Saving: (\\d\\d:\\d\\d.\\d\\d)");
				int savingTimeSeconds = -1;
				Instant timeStamp = null;
				Duration phaseDuration;	//We divide the job into 3 phases: preparation, rendering, compositing
				boolean scenePrepStarted = false;
				boolean renderingStarted = false;
				boolean postProcessingStarted = false;
				
				// Initialise the progress bar in the icon and the UI (0% completed at this time)
				gui.updateTrayIcon(0);
				gui.status("Preparing project", 0);
				
				while ((line = input.readLine()) != null) {
					log.debug(line);
					
					// Process lines until the version is loaded (usually first or second line of log)
					if (blenderLongVersion == null) {
						Pattern blenderPattern = Pattern.compile("Blender (([0-9]{1,3}\\.[0-9]{0,3}).*)$");
						Matcher blendDetectedVersion = blenderPattern.matcher(line);
						
						if (blendDetectedVersion.find()) {
							blenderLongVersion  = blendDetectedVersion.group(1);
							blenderShortVersion = blendDetectedVersion.group(2);
						}
					}
					
					Matcher scenePrepDetector = beginScenePrepPattern.matcher(line);
					if (scenePrepStarted == false && scenePrepDetector.find()) {
						scenePrepStarted = true;
						timeStamp = Instant.now();
					}
					
					progress = computeRenderingProgress(line, progressPattern, progress);
					if (renderingStarted == false && progress != -1) {
						renderingStarted = true;
						if (timeStamp == null) {
							timeStamp = new Date(process.getStartTime()).toInstant();
						}
						phaseDuration = Duration.between(timeStamp, Instant.now());
						timeStamp = Instant.now();
						process.setScenePrepDuration((int) phaseDuration.toSeconds());
					}
					
					Matcher postProcessingDetector = beginPostProcessingPattern.matcher(line);
					if (postProcessingStarted == false && postProcessingDetector.find()) {
						postProcessingStarted = true;
						if (timeStamp == null) {
							timeStamp = new Date(process.getStartTime()).toInstant();
						}
						phaseDuration = Duration.between(timeStamp, Instant.now());
						timeStamp = Instant.now();
						process.setRenderDuration((int) phaseDuration.toSeconds());
					}
					
					Matcher savingTimeDetector = savingPattern.matcher(line);
					if (savingTimeDetector.find()) {
						String savingTime = savingTimeDetector.group(1);
						if (savingTime != null) {
							savingTimeSeconds = (int) Duration.between(LocalTime.MIN, LocalTime.parse("00:" + savingTime)).toSeconds();	//add leading hours to comply with ISO time format
						}
					}
					
					if (configuration.getMaxAllowedMemory() != -1 && getProcessRender().getMemoryUsed().get() > configuration.getMaxAllowedMemory()) {
						log.debug("Blocking render because process ram used (" + getProcessRender().getMemoryUsed().get() + "k) is over user setting (" + configuration
								.getMaxAllowedMemory() + "k)");
						process.finish();
						if (process.getRenderDuration() == -1) {
							if (timeStamp == null) {
								timeStamp = new Date(process.getStartTime()).toInstant();
							}
							phaseDuration = Duration.between(timeStamp, Instant.now());
							process.setRenderDuration((int) phaseDuration.toSeconds());
						}
						if (script_file != null) {
							script_file.delete();
						}
						if (disableViewportScript != null) {
							disableViewportScript.delete();
						}
						
						// Once the process is finished (either finished successfully or with an error) move back to
						// base icon (isolated S with no progress bar)
						gui.updateTrayIcon(SHOW_BASE_ICON);
						
						return Error.Type.RENDERER_OUT_OF_MEMORY;
					}
					
					updateSpeedSamplesRendered(line);
					updateRenderingStatus(line, progress);
					Type error = detectError(line);
					if (error != Error.Type.OK) {
						if (script_file != null) {
							script_file.delete();
						}
						if (disableViewportScript != null) {
							disableViewportScript.delete();
						}
						if (process.getRenderDuration() == -1) {
							if (timeStamp == null) {
								timeStamp = new Date(process.getStartTime()).toInstant();
							}
							phaseDuration = Duration.between(timeStamp, Instant.now());
							process.setRenderDuration((int) phaseDuration.toSeconds());
						}
						
						// Put back base icon
						gui.updateTrayIcon(SHOW_BASE_ICON);
						process.kill();
						maybeCleanWorkingDir(error);
						for (String logline : configuration.filesystemHealthCheck()) {
							log.debug(logline);
						}
						
						return error;
					}
					
					if (!event.isStarted() && (getProcessRender().getMemoryUsed().get() > 0 && scenePrepStarted || process.getRemainingDuration() > 0)) {
						event.doNotifyIsStarted();
					}
				}
				
				if (timeStamp == null) {
					timeStamp = new Date(process.getStartTime()).toInstant();
				}
				
				if (postProcessingStarted == false) {
					phaseDuration = Duration.between(timeStamp, Instant.now());
					process.setRenderDuration((int) phaseDuration.toSeconds());
					
					//we need to subtract the time to save the frame to disk
					if (savingTimeSeconds > 0 && process.getRenderDuration() > 0) {
						process.setRenderDuration(Math.max(0, process.getRenderDuration() - savingTimeSeconds));
					}
				}
				else {
					phaseDuration = Duration.between(timeStamp, Instant.now());
					process.setPostProcessingDuration((int) phaseDuration.toSeconds());
				}
				input.close();
				
				log.debug(String.format("render times: %n\tScene prep: %ds%n\tRendering: %ds%n\tPost: %ss%n\tTotal: %ds%n\tRendering/Total: %.03f%n",
					process.getScenePrepDuration(),
					process.getRenderDuration(),
					process.getPostProcessingDuration(),
					process.getDuration(),
					(Math.max(process.getRenderDuration(), 0) * 100.0) / process.getDuration()
				));
			}
			catch (IOException err1) { // for the input.readline
				// most likely The handle is invalid
				log.error("Job::render exception(B) (silent error) " + err1);
			}
			finally {
				memoryCheck.cancel();
			}
			
			// Put back base icon
			gui.updateTrayIcon(SHOW_BASE_ICON);

			log.debug("end of rendering");
			
		}
		catch (Exception err) {
			if (script_file != null) {
				script_file.delete();
			}
			if (disableViewportScript != null) {
				disableViewportScript.delete();
			}
			StringWriter sw = new StringWriter();
			err.printStackTrace(new PrintWriter(sw));
			for (String logline : configuration.filesystemHealthCheck()) {
				log.debug(logline);
			}
			log.error("Job::render exception(A) " + err + " stacktrace " + sw.toString());
			return Error.Type.FAILED_TO_EXECUTE;
		}
		
		int exit_value = process.exitValue();
		process.finish();
		if (timerOfMaxRenderTime != null) {
			timerOfMaxRenderTime.cancel();
		}
		
		if (script_file != null) {
			script_file.delete();
		}
		if (disableViewportScript != null) {
			disableViewportScript.delete();
		}
		// find the picture file
		final String filename_without_extension = getPrefixOutputImage() + getFrameNumber();
		
		FilenameFilter textFilter = new FilenameFilter() {
			@Override public boolean accept(File dir, String name) {
				return name.startsWith(filename_without_extension);
			}
		};
		
		File[] files = configuration.getWorkingDirectory().listFiles(textFilter);
		
		if (isAskForRendererKill()) {
			log.debug("Job::render been asked to end render");
			
			long duration = (new Date().getTime() - process.getStartTime()) / 1000; // in seconds
			if (configuration.getMaxRenderTime() > 0 && duration > configuration.getMaxRenderTime() && Integer.parseInt(this.getId()) >= 20) {
				log.debug("Render killed because process duration (" + duration + "s) is over user setting (" + configuration.getMaxRenderTime() + "s)");
				return Error.Type.RENDERER_KILLED_BY_USER_OVER_TIME;
			}
			
			if (files.length != 0) {
				Arrays.stream(files).forEach( file -> new File(file.getAbsolutePath()).delete());
			}
			if (isServerBlockJob()) {
				return Error.Type.RENDERER_KILLED_BY_SERVER;
			}
			if (isUserBlockJob()) {
				return Error.Type.RENDERER_KILLED_BY_USER;
			}
			if (isIncompatibleProcessKillJob()) {
				return Error.Type.RENDERER_KILLED_BY_USER_INCOMPATIBLE_PROCESS;
			}
			return Error.Type.RENDERER_KILLED;
		}
		
		if (files.length == 0) {
			for (String logline : configuration.filesystemHealthCheck()) {
				log.debug(logline);
			}
			log.error("Job::render no picture file found (after finished render (filename_without_extension " + filename_without_extension + ")");
			
			String basename = "";
			try {
				basename = getPath().substring(0, getPath().lastIndexOf('.'));
			}
			catch (Exception e) {
				e.printStackTrace();
			}
			File crash_file = new File(configuration.getWorkingDirectory() + File.separator + basename + ".crash.txt");
			if (crash_file.exists()) {
				for (String logline : configuration.filesystemHealthCheck()) {
					log.debug(logline);
				}
				log.error("Job::render crash file found => the renderer crashed");
				crash_file.delete();
				return Error.Type.RENDERER_CRASHED;
			}
			
			if (exit_value == 127 && process.getDuration() < 10) {
				for (String logline : configuration.filesystemHealthCheck()) {
					log.debug(logline);
				}
				log.error("Job::render renderer returned 127 and took " + process.getDuration() + "s, some libraries may be missing");
				return Error.Type.RENDERER_MISSING_LIBRARIES;
			}
			
			return Error.Type.NOOUTPUTFILE;
		}
		else {
			if (files.length == 2) {
				Arrays.sort(files);	//in case of an exr we end up with 2 images, the output as an exr and the preview as a jpg, we want to ensure the output comes first
				String path = files[1].getAbsolutePath();
				String extension = path.substring(path.lastIndexOf(".") + 1).toLowerCase();
				if ("jpg".equals(extension)) {
					setPreviewImagePath(files[1].getAbsolutePath());
				}
			}
			
			setOutputImagePath(files[0].getAbsolutePath());
			this.outputImageSize = new File(getOutputImagePath()).length();
			log.debug(String.format("Job::render pictureFilename: %s, size: %d'", getOutputImagePath(), this.outputImageSize));
		}
		
		File scene_dir = new File(getSceneDirectory());
		long date_modification_scene_directory = (long) Utils.lastModificationTime(scene_dir);
		if (date_modification_scene_directory > process.getStartTime()) {
			scene_dir.delete();
		}
		
		gui.status(String.format("Render time: %dmin%ds",
			process.getRenderDuration() / 60,
			process.getRenderDuration() % 60)
		);
		
		return Error.Type.OK;
	}
	
	private int computeRenderingProgress(String line, Pattern tilePattern, int currentProgress) {
		Matcher standardTileInfo = tilePattern.matcher(line);
		int newProgress = currentProgress;
		
		if (standardTileInfo.find()) {
			int tileJustProcessed = Integer.parseInt(standardTileInfo.group(2));
			int totalTilesInJob = Integer.parseInt(standardTileInfo.group(3));
			
			newProgress = Math.abs((tileJustProcessed * 100) / totalTilesInJob);
		}
		
		// Only update the tray icon and the screen if percentage has changed
		if (newProgress != currentProgress) {
			gui.updateTrayIcon(newProgress);
			gui.status("Rendering", newProgress);
		}
		
		return newProgress;
	}
	
	private void updateSpeedSamplesRendered(String line) {
		// Looking for "Rendered 1281 samples in 66.319402 seconds"
		Pattern pattern = Pattern.compile("^Rendered (\\d+) samples in ([\\d.]+) seconds$");
		Matcher matcher = pattern.matcher(line);
		
		if (matcher.find()) {
			int amount = Integer.parseInt(matcher.group(1));
			float duration = Float.parseFloat(matcher.group(2));
			
			if (duration != 0 && amount != 0) {
				this.speedSamplesRendered = amount / duration;
			}
		}
		
		// should we use this, instead ???
		// "Average time per sample: 0.052112 seconds"
	}
	
	private void updateRenderingStatus(String line, int progress) {
		if (getUpdateRenderingStatusMethod() == null || UPDATE_METHOD_BY_REMAINING_TIME.equals(getUpdateRenderingStatusMethod())) {
			String search_remaining = "remaining:";
			int index = line.toLowerCase().indexOf(search_remaining);
			if (index != -1) {
				String buf1 = line.substring(index + search_remaining.length());
				index = buf1.indexOf(" ");
				
				if (index != -1) {
					String remaining_time = buf1.substring(0, index).trim();
					int last_index = remaining_time.lastIndexOf('.'); //format 00:00:00.00 (hr:min:sec)
					if (last_index > 0) {
						remaining_time = remaining_time.substring(0, last_index);
					}
					
					try {
						DateFormat date_parse_minute = new SimpleDateFormat("m:s");
						DateFormat date_parse_hour = new SimpleDateFormat("h:m:s");
						DateFormat date_parse = date_parse_minute;
						if (remaining_time.split(":").length > 2) {
							date_parse = date_parse_hour;
						}
						date_parse.setTimeZone(TimeZone.getTimeZone("GMT"));
						Date date = date_parse.parse(remaining_time);
						gui.setRemainingTime(Utils.humanDuration(date));
						getProcessRender().setRemainingDuration((int) (date.getTime() / 1000));
					}
					catch (ParseException err) {
						log.error("Client::updateRenderingStatus ParseException " + err);
					}
				}
			}
			else {	//extrapolate remaining time from time rendered & progress
				if (line.contains("Time") == true) {
					long timeRendered = new Date().getTime() - getProcessRender().getStartTime();
					
					if (progress > 0 && timeRendered > 0) {
						long linearTimeEstimation = (long) ((100.0 / progress) * timeRendered);
						long timeRemaining = linearTimeEstimation - timeRendered;
						Date date = new Date(timeRemaining);
						
						gui.setRemainingTime(Utils.humanDuration(date));
						getProcessRender().setRemainingDuration((int) (date.getTime() / 1000));
					}
				}
			}
		}
		else if (UPDATE_METHOD_BY_TILE.equals(getUpdateRenderingStatusMethod())) {
			String search = " Tile ";
			int index = line.lastIndexOf(search);
			if (index != -1) {
				String buf = line.substring(index + search.length());
				String[] parts = buf.split("/");
				if (parts.length == 2) {
					try {
						int current = Integer.parseInt(parts[0]);
						int total = Integer.parseInt(parts[1]);
						if (total != 0) {
							gui.status(String.format("Rendering %s %%", (int) (100.0 * current / total)));
							return;
						}
					}
					catch (NumberFormatException e) {
						System.out.println("Exception 94: " + e);
					}
				}
			}
			gui.status("Rendering");
		}
	}
	
	private void updateProcess() {
		getProcessRender().update();
	}
	
	private Type detectError(String line) {
		
		if (line.contains("version `GLIBC_2.28' not found")) {
			// 07-04 07:43:12 (debug) renderer output
			// 07-04 07:43:12 (debug) /tmp///rend.exe: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /tmp///rend.exe)
			// 07-04 07:43:12 (debug) /tmp///rend.exe: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /tmp///lib/libboost_filesystem.so.1.80.0)
			return Type.OS_NOT_SUPPORTED;
		}
		else if (line.contains("Color management:")) {
			String[] errors = {
				" not found",
				" is not compatible ",
				" Error could not find role",
				" Error, could not find role",
				"using fallback mode",
				" Error, could not find any displays",
				" Error, could not find any views",
				" Failed to load bundled config"
			};
			
			for (var error : errors) {
				if (line.contains(error)) {
			
					//30-11 01:16:57 (debug) Blender 4.0.0 (hash 878f71061b8e built 2023-11-14 01:20:37)
					//30-11 01:16:57 (debug) Color management: display "sRGB" used by scene not found, setting to default ("Blender").
					//30-11 01:16:57 (debug) Color management: scene view "AgX" not found, setting default "Standard".
					//30-11 01:16:57 (debug) Read blend: "C:\Users\tianji\AppData\Local\Temp\farm_3210908783605063546\254263927\Car Scene 3.blend"
					//30-11 01:16:57 (debug) Color management: display "sRGB" used by scene not found, setting to default ("Blender").
					//30-11 01:16:57 (debug) Color management: scene view "AgX" not found, setting default "Standard".
					//30-11 01:16:57 (debug) Color management: scene look "AgX - Medium High Contrast" not found, setting default "None".
					//30-11 01:16:57 (debug) Color management: image colorspace "Linear Rec.709" not found, will use default instead.
					return Type.COLOR_MANAGEMENT_ERROR;
				}
			}
		}
		else if (line.contains("OpenColorIO Error:")) {
			//28-11 12:43:40 (debug) Fra:340 Mem:11.74M (Peak 28.28M) | Time:00:01.69 | Compositing | Tile 1-1
			//28-11 12:43:40 (debug) Fra:340 Mem:11.68M (Peak 28.28M) | Time:00:01.69 | Compositing | De-initializing execution
			//28-11 12:43:40 (debug) OpenColorIO Error: The specified file reference 'srgb_inv.spi1d' could not be located. The following attempts were made: 'E:\SheepIt\sheepit\6b752e00cd23e1789e00285310cb6845\3.6\datafiles\colormanagement\luts\srgb_inv.spi1d' : 'E:\SheepIt\sheepit\6b752e00cd23e1789e00285310cb6845\3.6\datafiles\colormanagement\filmic\srgb_inv.spi1d'.
			//28-11 12:43:40 (debug) Saved: 'E:\SheepIt\sheepit\1_0340.png'
			return Type.COLOR_MANAGEMENT_ERROR;
		}
		else if (line.contains("CUDA error: Out of memory")) {
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.26M | Scene, RenderLayer | Updating Device | Writing constant memory
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.26M | Scene, RenderLayer | Path Tracing Tile 0/135, Sample 0/200
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.82M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 1/135, Sample 0/200
			// CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
			// Refer to the Cycles GPU rendering documentation for possible solutions:
			// http://www.blender.org/manual/render/cycles/gpu_rendering.html
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:09:26.57 | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 1/135, Sample 200/200
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.06 | Mem:470.50M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 134/135, Sample 0/200
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.03 | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 134/135, Sample 200/200
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.03 | Mem:470.50M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 135/135, Sample 0/200
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 135/135, Sample 200/200
			// Error: CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
			// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Cancel | CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
			// Fra:151 Mem:405.89M (0.00M, Peak 633.81M) Sce: Scene Ve:0 Fa:0 La:0
			// Saved: /tmp/xx/26885_0151.png Time: 00:04.67 (Saving: 00:00.22)
			// Blender quit
			return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
		}
		else if (line.contains("CUDA error at cuCtxCreate: Out of memory")) {
			// renderer output
			// CUDA error at cuCtxCreate: Out of memory
			// Refer to the Cycles GPU rendering documentation for possible solutions:
			// http://www.blender.org/manual/render/cycles/gpu_rendering.html
			// found bundled python: /tmp/aaaa/bbbb/2.78/python
			// read blend: /tmp/aaaa/bbbb/compute-method.blend
			// Fra:340 Mem:7.25M (0.00M, Peak 7.25M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun
			// Fra:340 Mem:7.25M (0.00M, Peak 7.25M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Plane
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Cube
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Camera
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Loading render kernels (may take a few minutes the first time)
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Error | CUDA error at cuCtxCreate: Out of memory
			// Error: CUDA error at cuCtxCreate: Out of memory
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Waiting for render to start
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Cancel | CUDA error at cuCtxCreate: Out of memory
			// CUDA error: Invalid value in cuCtxDestroy(cuContext)
			// Fra:340 Mem:7.25M (0.00M, Peak 7.26M) | Time:00:00.13 | Sce: Scene Ve:0 Fa:0 La:0
			// Blender quit
			// end of rendering
			return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
		}
		else if (line.contains("Error: Out of memory in CUDA") || line.contains("Error: Out of memory in OPTIX")) {
			// 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.27 | Mem:8411.20M, Peak:8411.20M | Scene, View Layer | Loading render kernels (may take a few minutes the first time)
			// 15-03 21:13:45 (debug) Out of memory in CUDA queue enqueue (integrator_shade_surface)
			// 15-03 21:13:45 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
			// 15-03 21:13:45 (debug) https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html
			// 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.38 | Mem:8411.27M, Peak:8411.27M | Scene, View Layer | Updating Scene
			// 15-03 21:13:45 (debug) Fra:27 Mem:8441.68M (Peak 9675.81M) | Time:00:16.38 | Mem:8411.27M, Peak:8411.27M | Scene, View Layer | Updating Shaders
			// 15-03 21:13:45 (debug) Fra:27 Mem:8447.08M (Peak 9675.81M) | Time:00:16.63 | Mem:8763.00M, Peak:8763.00M | Scene, View Layer | Out of memory in CUDA queue enqueue (integrator_shade_surface)
			// 15-03 21:13:45 (debug) Error: Out of memory in CUDA queue enqueue (integrator_shade_surface)
			// 15-03 21:13:46 (debug) Blender quit
			return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
		}
		else if (line.contains("CUDA error: Launch exceeded timeout in") || line.contains("CUDA error: Invalid value in cuTexRefSetAddress(") || line.contains("CUDA error: Launch failed in cuCtxSynchronize()") || line.contains("CUDA error: Illegal address in cuCtxSynchronize()")) {
			return Error.Type.GPU_OR_DRIVER_ERROR;
		}
		else if (line.contains("System is out of GPU memory")) {
			// 16-03 18:37:37 (debug) Fra:192 Mem:15826.70M (Peak 15831.18M) | Time:02:21.86 | Mem:6622.87M, Peak:6625.35M | Scene, ViewLayer | Loading denoising kernels (may take a few minutes the first time)
			// 16-03 18:37:37 (debug) System is out of GPU memory
			// 16-03 18:37:37 (debug) Refer to the Cycles GPU rendering documentation for possible solutions:
			// 16-03 18:37:37 (debug) https://docs.blender.org/manual/en/latest/render/cycles/gpu_rendering.html
			// 16-03 18:37:37 (debug) System is out of GPU memory
			// 16-03 18:37:38 (debug) Fra:192 Mem:15831.01M (Peak 15831.18M) | Time:02:23.11 | Mem:7017.46M, Peak:7017.46M | Scene, ViewLayer | System is out of GPU memory
			// 16-03 18:37:38 (debug) Error: System is out of GPU memory
			// 16-03 18:37:40 (debug) Blender quit
			// 16-03 18:37:40 (debug) end of rendering
			return Error.Type.RENDERER_OUT_OF_VIDEO_MEMORY;
		}
		else if (line.contains("CUDA device supported only with compute capability") || line.contains("OPTIX device supported only with compute capability")) {
			// found bundled python: /tmp/xx/2.73/python
			// read blend: /tmp/xx/compute-method.blend
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Plane
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Cube
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Camera
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Loading render kernels (may take a few minutes the first time)
			// CUDA device supported only with compute capability 2.0 or up, found 1.2.
			// Refer to the Cycles GPU rendering documentation for possible solutions:
			// http://www.blender.org/manual/render/cycles/gpu_rendering.html
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Error | CUDA device supported only with compute capability 2.0 or up, found 1.2.
			// Error: CUDA device supported only with compute capability 2.0 or up, found 1.2.
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Waiting for render to start
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Cancel | CUDA device supported only with compute capability 2.0 or up, found 1.2.
			// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) Sce: Scene Ve:0 Fa:0 La:0
			// Saved: /tmp/xx/0_0340.png Time: 00:00.12 (Saving: 00:00.03)
			// Blender quit
			return Type.GPU_NOT_SUPPORTED;
		}
		else if (line.contains("terminate called after throwing an instance of 'boost::filesystem::filesystem_error'")) {
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.24 | Mem:1.64M, Peak:1.64M | Scene, RenderLayer | Updating Mesh | Computing attributes
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.24 | Mem:1.64M, Peak:1.64M | Scene, RenderLayer | Updating Mesh | Copying Attributes to device
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.24 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Building
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.24 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Building BVH
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.24 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Looking in BVH cache
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.27 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Packing BVH triangles and strands
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.27 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Packing BVH nodes
			// Fra:2103 Mem:29.54M (0.00M, Peak 29.54M) | Time:00:00.27 | Mem:1.97M, Peak:1.97M | Scene, RenderLayer | Updating Scene BVH | Writing BVH cache
			// terminate called after throwing an instance of 'boost::filesystem::filesystem_error'
			//   what():  boost::filesystem::create_directory: Permission denied: "/var/local/cache"
			return Error.Type.NOOUTPUTFILE;
		}
		else if (line.contains("terminate called after throwing an instance of 'std::bad_alloc'")) {
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Mesh BVH Plane.083 171/2 | Building BVH
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Mesh BVH Mesh 172/2 | Building BVH
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Mesh BVH Mesh 172/2 | Packing BVH triangles and strands
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Mesh BVH Mesh 172/2 | Packing BVH nodes
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Scene BVH | Building
			// Fra:80 Mem:1333.02M (0.00M, Peak 1651.23M) | Mem:780.37M, Peak:780.37M | Scene, RenderLayer | Updating Scene BVH | Building BVH
			// terminate called after throwing an instance of 'std::bad_alloc'
			//   what():  std::bad_alloc
			return Error.Type.RENDERER_OUT_OF_MEMORY;
		}
		else if (line.contains("what(): std::bad_alloc")) {
			// Fra:7 Mem:1247.01M (0.00M, Peak 1247.01M) | Time:00:28.84 | Mem:207.63M, Peak:207.63M | Scene, RenderLayer | Updating Scene BVH | Building BVH 93%, duplicates 0%terminate called recursively
			// terminate called after throwing an instance of 'St9bad_alloc'
			// what(): std::bad_alloc
			// scandir: Cannot allocate memory
			return Error.Type.RENDERER_OUT_OF_MEMORY;
		}
		else if (line.contains("Fatal Python error:")) {
			// Fatal Python error: Py_Initialize: unable to load the file system codec
			// ImportError: No module named 'encodings'
			// Current thread 0x0000388c (most recent call first):
			return Error.Type.RENDERER_CRASHED_PYTHON_ERROR;
		}
		else if (line.contains("Calloc returns null")) {
			// Fra:1 Mem:976.60M (0.00M, Peak 1000.54M) | Time:00:01.34 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Left
			// Calloc returns null: len=7186416 in CDMLoopUV, total 2145859048
			// Calloc returns null: len=7186416 in CDMLoopUV, total 2145859048
			// Malloc returns null: len=3190672 in CDMTexPoly, total 2149293176
			// Writing: /home/user/.sheepit/LEFT packed.crash.txt
			return Error.Type.RENDERER_OUT_OF_MEMORY;
		}
		else if (line.contains("Malloc returns null")) {
			// Fra:1 Mem:976.60M (0.00M, Peak 1000.54M) | Time:00:01.34 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Left
			// Calloc returns null: len=7186416 in CDMLoopUV, total 2145859048
			// Calloc returns null: len=7186416 in CDMLoopUV, total 2145859048
			// Malloc returns null: len=3190672 in CDMTexPoly, total 2149293176
			// Writing: /home/user/.sheepit/LEFT packed.crash.txt
			return Error.Type.RENDERER_OUT_OF_MEMORY;
		}
		else if (line.contains("Error: run out of memory!")) {
			// 09-01 09:26:45 (debug) Fra:4 Mem:8444.44M (Peak 12092.10M) | Time:01:11.45 | Mem:6632.02M, Peak:6632.02M | Scene, ViewLayer | Updating Mesh | Tessellating Plane 1/1
			//09-01 09:27:30 (debug) Error: run out of memory!
			//09-01 09:27:30 (debug) Fra:4 Mem:8444.44M (Peak 12092.10M) | Time:01:55.95 | Mem:6632.02M, Peak:6632.02M | Scene, ViewLayer | Sample 0/1024
			//09-01 09:27:30 (debug) Error: Out of memory
			//09-01 09:27:30 (debug) Fra:4 Mem:8745.04M (Peak 12092.10M) | Time:01:55.97 | Mem:6632.03M, Peak:6632.03M | Scene, ViewLayer | Out of memory
			//09-01 09:27:33 (debug) Blender quit
			return Error.Type.RENDERER_OUT_OF_MEMORY;
		}
		else if (line.contains("CUDA kernel compilation failed") || line.contains("OPTIX kernel compilation failed")) {
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.001
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun.002
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.02 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Loading render kernels (may take a few minutes the first time)
			// nvcc fatal   : Value 'sm_61' is not defined for option 'gpu-architecture'
			// CUDA kernel compilation failed, see console for details.
			// Refer to the Cycles GPU rendering documentation for possible solutions:
			// http://www.blender.org/manual/render/cycles/gpu_rendering.html
			// Compiling CUDA kernel ...
			// "nvcc" -arch=sm_61 -m64 --cubin "/tmp/cache/c36db40aa5e59f5ea4ff139180353dbc/2.77/scripts/addons/cycles/kernel/kernels/cuda/kernel.cu" -o "/tmp/cache/cycles_kernel_sm61_079195D400571E023CC499D037AB6EE5.cubin" --ptxas-options="-v" --use_fast_math -I"/tmp/cache/c36db40aa5e59f5ea4ff139180353dbc/2.77/scripts/addons/cycles/kernel" -DNVCC -D__KERNEL_CUDA_VERSION__=75
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.08 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Error | CUDA kernel compilation failed, see console for details.
			// Error: CUDA kernel compilation failed, see console for details.
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.08 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Waiting for render to start
			// Fra:1 Mem:200.70M (0.00M, Peak 378.15M) | Time:00:01.08 | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Cancel | CUDA kernel compilation failed, see console for details.
			// Fra:1 Mem:147.74M (0.00M, Peak 378.15M) | Time:00:01.12 | Sce: Scene Ve:0 Fa:0 La:0
			// Saved: '/tmp/cache/8_0001.png'
			return Error.Type.GPU_NOT_SUPPORTED;
		}
		else if (line.contains("Engine 'CYCLES' not available for scene") || line.contains("Engine 'BLENDER_EEVEE' not available for scene") || line.contains("Engine 'BLENDER_WORKBENCH' not available for scene") || line.contains("Engine 'HYDRA_STORM' not available for scene")) {
			return Error.Type.ENGINE_NOT_AVAILABLE;
		}
		else if (line.contains("Warning: Cycles is not enabled!")) {
			return Error.Type.ENGINE_NOT_AVAILABLE;
		}
		else if (line.contains("OpenImageDenoise error: SSE4.1 support is required at minimum") || line.contains("OpenImageDenoiser is not supported on this CPU:") || line.contains("OpenImageDenoise is not supported on this CPU:") || line.contains("No device available to denoise on")) {
			//OpenImageDenoiser is Blender pre 4.2
			//OpenImageDenoise after Blender 4.2
			
			// denoising capability detection
			// Fra:201 Mem:8.89M (Peak 10.09M) | Time:00:00.04 | Mem:0.27M, Peak:0.27M | Scene, View Layer | Updating Device | Writing constant memory
			// Fra:201 Mem:8.89M (Peak 10.09M) | Time:00:00.04 | Mem:0.27M, Peak:0.27M | Scene, View Layer | Loading denoising kernels (may take a few minutes the first time)
			// Fra:201 Mem:8.89M (Peak 10.09M) | Time:00:00.04 | Mem:0.27M, Peak:0.27M | Scene, View Layer | Sample 0/16
			// Fra:201 Mem:127.55M (Peak 127.55M) | Time:00:00.65 | Remaining:00:09.13 | Mem:118.92M, Peak:118.92M | Scene, View Layer | Sample 1/16
			// E0308 1719 31449 denoiser_oidn.cpp:197] OpenImageDenoise error: SSE4.1 support is required at minimum
			// Fra:201 Mem:198.74M (Peak 301.58M) | Time:00:08.74 | Mem:118.92M, Peak:118.92M | Scene, View Layer | Sample 16/16
			// Rendering done
			// Fra:201 Mem:198.74M (Peak 301.58M) | Time:00:08.74 | Mem:118.92M, Peak:118.92M | Scene, View Layer | Finished
			// Saved: '/tmp/5/sheepit/1541_0201.png'
			// Time: 00:09.30 (Saving: 00:00.55)
			// Blender quit
			return Error.Type.DENOISING_NOT_SUPPORTED;
		}
		else if (line.contains("Error: File format is not supported")) {
			//09-03 23:36:03 (debug) Blender 2.83.19 (hash 86c526d2c733 built 2022-02-02 00:44:40)
			//09-03 23:36:03 (debug) found bundled python: C:\Users\Username\Documents\sheepit\sheepit\f1f9e587201b428c9416faf614b677bf\2.83\python
			//09-03 23:36:03 (debug) Error: File format is not supported in file 'C:\Users\Username\Documents\sheepit\sheepit\686c6bdbf752790b28c1b21180aa5db5\correndo4.blend'
			//09-03 23:36:03 (debug) Blender quit
			//09-03 23:36:03 (debug) end of rendering
			return Error.Type.RENDERER_CRASHED;
		}
		else if (line.contains("Unable to open a display") || line.contains("Managed to successfully fallback to surfaceless EGL rendering!") || line.contains("Couldn't open libEGL")) {
			//17-03 07:38:47 (debug) Fra:7 Mem:1667.27M (Peak 1762.19M) | Time:06:27.86 | Mem:1184.92M, Peak:1194.58M | Scene, ViewLayer | Sample 187/187
			//17-03 07:38:47 (debug) Fra:7 Mem:1667.27M (Peak 1762.19M) | Time:06:27.86 | Mem:1184.92M, Peak:1194.58M | Scene, ViewLayer | Finished
			//17-03 07:38:47 (debug) Unable to open a display
			//17-03 07:38:47 (debug) end of rendering
			//17-03 07:38:47 (error) Job::render no picture file found (after finished render (filename_without_extension 116372253_0007)
			return Error.Type.CURRENTLY_HEADLESS;
		}
		else if (line.contains("Error: Cannot read file") || line.contains("Error: Failed to read blend file")) {
			//14-10 11:03:21 (debug) Blender 3.1.0 (hash c77597cd0e15 built 2022-03-09 00:44:13)
			//14-10 11:03:21 (debug) Error: Cannot read file 'C:\SheepIT\sheepit\ef925644a319c5ad604ebc077769bcb8\Passeio Virtual FINAL.blend': No such file or directory
			//14-10 11:03:21 (debug) Blender quit
			return Error.Type.CANNOT_READ_FILE;
		}
		else if (line.contains("DETECT_DEVICE_ERROR")) {
			//./blender-3.3.2-linux-x64/blender -b -E CYCLES -P /mnt/d/sheep_gpu/set_gpu.py -f 1
			//Blender 3.3.2 (hash bf24652e38a2 built 2022-12-07 01:12:36)
			//Read prefs: /home/raimund/.config/blender/3.3/config/userpref.blend
			///run/user/1000/gvfs/ non-existent directory
			//DETECT_DEVICE_ERRORR: Couldn't find OPTIX device with id CUDA_NVIDIA GeForce GTX 1080_0000:01:00_OptiX
			return Error.Type.DETECT_DEVICE_ERROR;
		}

		return Type.OK;
	}
	
	private void maybeCleanWorkingDir(Type error) {
		boolean cleanup = Type.COLOR_MANAGEMENT_ERROR == error
			|| Type.RENDERER_CRASHED_PYTHON_ERROR == error
			|| Type.ENGINE_NOT_AVAILABLE == error
			|| Type.DETECT_DEVICE_ERROR == error;
		if (cleanup) {
			configuration.cleanWorkingDirectory();
		}
	}
	
	public static class renderStartedObservable extends Observable {
		
		@Getter private boolean isStarted;
		
		public renderStartedObservable(Observer observer) {
			super();
			addObserver(observer);
		}
		
		public void doNotifyIsStarted() {
			setChanged();
			notifyObservers();
			isStarted = true;
		}
	}
}