Ref: cleanup, organise packages
This commit is contained in:
Sheepit Renderfarm
803
src/main/java/com/sheepit/client/rendering/Job.java
Normal file
803
src/main/java/com/sheepit/client/rendering/Job.java
Normal file
@@ -0,0 +1,803 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2014 Laurent CLOUET
|
||||
* Author Laurent CLOUET <laurent.clouet@nopnop.net>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; version 2
|
||||
* of the License.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
package com.sheepit.client.rendering;
|
||||
|
||||
import com.sheepit.client.logger.Log;
|
||||
import com.sheepit.client.config.Configuration;
|
||||
import com.sheepit.client.config.Configuration.ComputeType;
|
||||
import com.sheepit.client.config.DirectoryManager;
|
||||
import com.sheepit.client.datamodel.client.Error;
|
||||
import com.sheepit.client.datamodel.client.Error.Type;
|
||||
import com.sheepit.client.network.DownloadItem;
|
||||
import com.sheepit.client.os.OS;
|
||||
import com.sheepit.client.ui.Gui;
|
||||
import com.sheepit.client.utils.Utils;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Observable;
|
||||
import java.util.Observer;
|
||||
import java.util.Optional;
|
||||
import java.util.TimeZone;
|
||||
import java.util.Timer;
|
||||
import java.util.TimerTask;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.sheepit.client.rendering.RenderSettings.UPDATE_METHOD_BY_REMAINING_TIME;
|
||||
import static com.sheepit.client.rendering.RenderSettings.UPDATE_METHOD_BY_TILE;
|
||||
|
||||
@Data public class Job {
|
||||
public static final String POST_LOAD_NOTIFICATION = "POST_LOAD_SCRIPT_loaded";
|
||||
|
||||
public static final int SHOW_BASE_ICON = -1;
|
||||
|
||||
private DownloadItem projectDownload;
|
||||
private DownloadItem rendererDownload;
|
||||
private String id;
|
||||
private String validationUrl;
|
||||
private String name;
|
||||
private char[] password;
|
||||
|
||||
private boolean synchronousUpload;
|
||||
|
||||
private RenderSettings renderSettings;
|
||||
private RenderProcess renderProcess;
|
||||
private RenderOutput renderOutput;
|
||||
private RenderState renderState;
|
||||
|
||||
private Gui gui;
|
||||
private Configuration configuration;
|
||||
private Log log;
|
||||
|
||||
private DirectoryManager directoryManager;
|
||||
|
||||
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
|
||||
String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
|
||||
String update_method_) {
|
||||
configuration = config_;
|
||||
id = id_;
|
||||
renderSettings = new RenderSettings(frame_, script_, path_, command_, use_gpu, update_method_);
|
||||
validationUrl = validationUrl_;
|
||||
projectDownload = projectDownload_;
|
||||
rendererDownload = rendererDownload_;
|
||||
name = name_;
|
||||
password = password_.clone();
|
||||
synchronousUpload = synchronous_upload_;
|
||||
gui = gui_;
|
||||
log = log_;
|
||||
directoryManager = new DirectoryManager(configuration);
|
||||
renderProcess = new RenderProcess(log_);
|
||||
renderState = new RenderState();
|
||||
renderOutput = new RenderOutput();
|
||||
}
|
||||
|
||||
public void block() {
|
||||
renderState.setBlock();
|
||||
|
||||
RenderProcess process = getProcessRender();
|
||||
if (process != null) {
|
||||
process.kill();
|
||||
}
|
||||
}
|
||||
|
||||
public void incompatibleProcessBlock() {
|
||||
renderState.setBlockIncompatibleProcess();
|
||||
RenderProcess process = getProcessRender();
|
||||
if (process != null) {
|
||||
process.kill();
|
||||
}
|
||||
}
|
||||
|
||||
public RenderProcess getProcessRender() {
|
||||
return renderProcess;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String
|
||||
.format("Job (numFrame '%s' archiveChunks %s rendererMD5 '%s' ID '%s' pictureFilename '%s' jobPath '%s' gpu %s name '%s' updateRenderingStatusMethod '%s' render %s)",
|
||||
getRenderSettings().getFrameNumber(), projectDownload.getMD5(), rendererDownload.getMD5(), id, getRenderOutput().getFullImagePath(),
|
||||
getRenderSettings().getPath(), getRenderSettings().isUseGPU(), name, getRenderSettings().getUpdateRenderingStatusMethod(), renderProcess);
|
||||
}
|
||||
|
||||
public String getPrefixOutputImage() {
|
||||
return id + "_";
|
||||
}
|
||||
|
||||
public String getRendererDirectory() {
|
||||
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererDownload.getMD5();
|
||||
}
|
||||
|
||||
public String getRendererPath() {
|
||||
return getRendererDirectory() + File.separator + OS.getOS().getRenderBinaryPath();
|
||||
}
|
||||
|
||||
public String getSceneDirectory() {
|
||||
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id;
|
||||
}
|
||||
|
||||
public String getScenePath() {
|
||||
return getSceneDirectory() + File.separator + getRenderSettings().getPath();
|
||||
}
|
||||
|
||||
public Error.Type render(Observer renderStarted) {
|
||||
gui.status("Rendering");
|
||||
RenderProcess process = getProcessRender();
|
||||
Timer timerOfMaxRenderTime = null;
|
||||
String core_script;
|
||||
// When sending Ctrl+C to the terminal it also get's sent to all subprocesses e.g. also the render process.
|
||||
// The java program handles Ctrl+C but the renderer quits on Ctrl+C.
|
||||
// This script causes the renderer to ignore Ctrl+C.
|
||||
String ignore_signal_script = "import signal\n" + "def hndl(signum, frame):\n" + " pass\n" + "signal.signal(signal.SIGINT, hndl)\n";
|
||||
if (getRenderSettings().isUseGPU() && configuration.getGPUDevice() != null && configuration.getComputeMethod() != ComputeType.CPU) {
|
||||
core_script = "sheepit_set_compute_device(\"" + configuration.getGPUDevice().getType() + "\", \"GPU\", \"" + configuration.getGPUDevice().getId()
|
||||
+ "\")\n";
|
||||
gui.setComputeMethod("GPU");
|
||||
}
|
||||
else {
|
||||
// Otherwise (CPU), fix the tile size to 32x32px
|
||||
core_script = "sheepit_set_compute_device(\"NONE\", \"CPU\", \"CPU\")\n";
|
||||
gui.setComputeMethod("CPU");
|
||||
}
|
||||
|
||||
core_script += ignore_signal_script;
|
||||
File disableViewportScript = null;
|
||||
File script_file = null;
|
||||
String[] command1 = getRenderSettings().getCommand().split(" ");
|
||||
int size_command = command1.length + 2; // + 2 for script
|
||||
|
||||
if (configuration.getNbCores() > 0) { // user has specified something
|
||||
size_command += 2;
|
||||
}
|
||||
|
||||
List<String> command = new ArrayList<>(size_command);
|
||||
|
||||
Map<String, String> new_env = new HashMap<>();
|
||||
|
||||
//make sure the system doesn´t interfere with the blender runtime, and that blender doesn´t attempt to load external libraries/scripts.
|
||||
new_env.put("BLENDER_USER_RESOURCES", "");
|
||||
new_env.put("BLENDER_USER_CONFIG", "");
|
||||
new_env.put("BLENDER_USER_EXTENSIONS", "");
|
||||
new_env.put("BLENDER_USER_SCRIPTS", "");
|
||||
new_env.put("BLENDER_USER_DATAFILES", "");
|
||||
|
||||
new_env.put("BLENDER_SYSTEM_RESOURCES", "");
|
||||
new_env.put("BLENDER_SYSTEM_SCRIPTS", "");
|
||||
new_env.put("BLENDER_SYSTEM_DATAFILES", "");
|
||||
new_env.put("BLENDER_SYSTEM_PYTHON", "");
|
||||
|
||||
new_env.put("OCIO", ""); //prevent blender from loading a non-standard color configuration
|
||||
new_env.put("TEMP", configuration.getWorkingDirectory().getAbsolutePath().replace("\\", "\\\\"));
|
||||
new_env.put("TMP", configuration.getWorkingDirectory().getAbsolutePath().replace("\\", "\\\\"));
|
||||
|
||||
new_env.put("CORES", Integer.toString(configuration.getNbCores()));
|
||||
new_env.put("PRIORITY", Integer.toString(configuration.getPriority()));
|
||||
|
||||
// Add own lib folder first, because Steam or other environments may set an LD_LIBRARY_PATH that has priority over the runpath in the Blender excutable,
|
||||
// but contains incompatible libraries.
|
||||
String currentLDLibraryPath = Optional.ofNullable(System.getenv("LD_LIBRARY_PATH")).orElse("");
|
||||
new_env.put("LD_LIBRARY_PATH", getRendererDirectory() + "/lib" + ":" + currentLDLibraryPath);
|
||||
|
||||
for (String arg : command1) {
|
||||
switch (arg) {
|
||||
case ".c":
|
||||
command.add("-P");
|
||||
|
||||
try {
|
||||
disableViewportScript = File.createTempFile("pre_load_script_", ".py", configuration.getWorkingDirectory());
|
||||
File file = new File(disableViewportScript.getAbsolutePath());
|
||||
FileWriter fwriter;
|
||||
fwriter = new FileWriter(file);
|
||||
|
||||
PrintWriter out = new PrintWriter(fwriter);
|
||||
out.write("import bpy");
|
||||
out.write("\n");
|
||||
out.write("import sys");
|
||||
out.write("\n");
|
||||
out.write("from bpy.app.handlers import persistent");
|
||||
out.write("\n");
|
||||
out.write("@persistent");
|
||||
out.write("\n");
|
||||
out.write("def hide_stuff(hide_dummy):");
|
||||
out.write("\n");
|
||||
out.write(" print('PRE_LOAD_SCRIPT_hide_viewport')");
|
||||
out.write("\n");
|
||||
out.write(" #Hide collections in the viewport");
|
||||
out.write("\n");
|
||||
out.write(" for col in bpy.data.collections:");
|
||||
out.write("\n");
|
||||
out.write(" col.hide_viewport = True");
|
||||
out.write("\n");
|
||||
out.write(" for obj in bpy.data.objects:");
|
||||
out.write("\n");
|
||||
out.write(" #Hide objects in the viewport");
|
||||
out.write("\n");
|
||||
out.write(" #obj.hide_viewport = True");
|
||||
out.write("\n");
|
||||
out.write(" #Hide modifier in the viewport");
|
||||
out.write("\n");
|
||||
out.write(" for mod in obj.modifiers:");
|
||||
out.write("\n");
|
||||
out.write(" mod.show_viewport = False");
|
||||
out.write("\n");
|
||||
out.write(" sys.stdout.flush()");
|
||||
out.write("\n");
|
||||
out.write("bpy.app.handlers.version_update.append(hide_stuff)");
|
||||
out.write("\n");
|
||||
out.close();
|
||||
|
||||
command.add(disableViewportScript.getAbsolutePath());
|
||||
}
|
||||
catch (IOException e) {
|
||||
StringWriter sw = new StringWriter();
|
||||
e.printStackTrace(new PrintWriter(sw));
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
||||
return Error.Type.UNKNOWN;
|
||||
}
|
||||
|
||||
command.add(getScenePath());
|
||||
command.add("-P");
|
||||
|
||||
try {
|
||||
script_file = File.createTempFile("post_load_script_", ".py", configuration.getWorkingDirectory());
|
||||
File file = new File(script_file.getAbsolutePath());
|
||||
FileWriter txt;
|
||||
txt = new FileWriter(file);
|
||||
|
||||
PrintWriter out = new PrintWriter(txt);
|
||||
out.write(getRenderSettings().getScript());
|
||||
out.write("\n");
|
||||
out.write("import sys");
|
||||
out.write("\n");
|
||||
out.write("print('" + POST_LOAD_NOTIFICATION + "')");
|
||||
out.write("\n");
|
||||
out.write("sys.stdout.flush()");
|
||||
out.write("\n");
|
||||
out.write(core_script); // GPU part
|
||||
out.write("\n"); // GPU part
|
||||
out.close();
|
||||
|
||||
command.add(script_file.getAbsolutePath());
|
||||
}
|
||||
catch (IOException e) {
|
||||
StringWriter sw = new StringWriter();
|
||||
e.printStackTrace(new PrintWriter(sw));
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
||||
return Error.Type.UNKNOWN;
|
||||
}
|
||||
script_file.deleteOnExit();
|
||||
break;
|
||||
case ".e":
|
||||
command.add(getRendererPath());
|
||||
// the number of cores has to be put after the binary and before the scene arg
|
||||
if (configuration.getNbCores() > 0) {
|
||||
command.add("-t");
|
||||
command.add(Integer.toString(configuration.getNbCores()));
|
||||
}
|
||||
break;
|
||||
case ".o":
|
||||
command.add(configuration.getWorkingDirectory().getAbsolutePath() + File.separator + getPrefixOutputImage());
|
||||
break;
|
||||
case ".f":
|
||||
command.add(getRenderSettings().getFrameNumber());
|
||||
break;
|
||||
default:
|
||||
command.add(arg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Timer memoryCheck = new Timer();
|
||||
try {
|
||||
renderStartedObservable event = new renderStartedObservable(renderStarted);
|
||||
String line;
|
||||
log.debug(command.toString());
|
||||
OS os = OS.getOS();
|
||||
process.setCoresUsed(configuration.getNbCores());
|
||||
process.start();
|
||||
getProcessRender().setProcess(os.exec(command, new_env));
|
||||
getProcessRender().setOsProcess(OS.operatingSystem.getProcess((int) getProcessRender().getProcess().pid()));
|
||||
BufferedReader input = new BufferedReader(new InputStreamReader(getProcessRender().getProcess().getInputStream()));
|
||||
memoryCheck.scheduleAtFixedRate(new TimerTask() {
|
||||
@Override public void run() {
|
||||
updateProcess();
|
||||
}
|
||||
}, 0L, 200L);
|
||||
|
||||
// Make initial test/power frames ignore the maximum render time in user configuration. Initial test frames have Job IDs below 20
|
||||
// so we just activate the user defined timeout when the scene is not one of the initial ones.
|
||||
if (configuration.getMaxRenderTime() > 0 && Integer.parseInt(this.getId()) >= 20) {
|
||||
timerOfMaxRenderTime = new Timer();
|
||||
timerOfMaxRenderTime.schedule(new TimerTask() {
|
||||
@Override public void run() {
|
||||
RenderProcess process = getProcessRender();
|
||||
if (process != null) {
|
||||
long duration = (new Date().getTime() - process.getStartTime()) / 1000; // in seconds
|
||||
if (configuration.getMaxRenderTime() > 0 && duration > configuration.getMaxRenderTime()) {
|
||||
getRenderState().setAskForRendererKill(true);
|
||||
log.debug("Killing render - exceeding allowed process duration");
|
||||
process.kill();
|
||||
}
|
||||
}
|
||||
}
|
||||
}, configuration.getMaxRenderTime() * 1000 + 2000); // +2s to be sure the delay is over
|
||||
}
|
||||
|
||||
|
||||
log.debug("renderer output");
|
||||
try {
|
||||
int progress = -1;
|
||||
|
||||
Pattern progressPattern = Pattern.compile(" (Rendered|Path Tracing Tile|Rendering|Sample) (\\d+)\\s?\\/\\s?(\\d+)( Tiles| samples|,)*");
|
||||
Pattern beginScenePrepPattern = Pattern.compile(POST_LOAD_NOTIFICATION);
|
||||
Pattern beginPostProcessingPattern = Pattern.compile("^Fra:\\d* \\w*(.)* \\| (Compositing|Denoising)");
|
||||
Pattern savingPattern = Pattern.compile("Time: \\d\\d:\\d\\d.\\d\\d \\(Saving: (\\d\\d:\\d\\d.\\d\\d)");
|
||||
int savingTimeSeconds = -1;
|
||||
Instant timeStamp = null;
|
||||
Duration phaseDuration; //We divide the job into 3 phases: preparation, rendering, compositing
|
||||
boolean scenePrepStarted = false;
|
||||
boolean renderingStarted = false;
|
||||
boolean postProcessingStarted = false;
|
||||
|
||||
// Initialise the progress bar in the icon and the UI (0% completed at this time)
|
||||
gui.updateTrayIcon(0);
|
||||
gui.status("Preparing project", 0);
|
||||
|
||||
while ((line = input.readLine()) != null) {
|
||||
log.debug(line);
|
||||
|
||||
// Process lines until the version is loaded (usually first or second line of log)
|
||||
if (getRenderOutput().getBlenderLongVersion() == null) {
|
||||
Pattern blenderPattern = Pattern.compile("Blender (([0-9]{1,3}\\.[0-9]{0,3}).*)$");
|
||||
Matcher blendDetectedVersion = blenderPattern.matcher(line);
|
||||
|
||||
if (blendDetectedVersion.find()) {
|
||||
getRenderOutput().setBlenderLongVersion(blendDetectedVersion.group(1));
|
||||
getRenderOutput().setBlenderShortVersion(blendDetectedVersion.group(2));
|
||||
}
|
||||
}
|
||||
|
||||
Matcher scenePrepDetector = beginScenePrepPattern.matcher(line);
|
||||
if (scenePrepStarted == false && scenePrepDetector.find()) {
|
||||
scenePrepStarted = true;
|
||||
timeStamp = Instant.now();
|
||||
}
|
||||
|
||||
progress = computeRenderingProgress(line, progressPattern, progress);
|
||||
if (renderingStarted == false && progress != -1) {
|
||||
renderingStarted = true;
|
||||
if (timeStamp == null) {
|
||||
timeStamp = new Date(process.getStartTime()).toInstant();
|
||||
}
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
timeStamp = Instant.now();
|
||||
process.setScenePrepDuration((int) phaseDuration.toSeconds());
|
||||
}
|
||||
|
||||
Matcher postProcessingDetector = beginPostProcessingPattern.matcher(line);
|
||||
if (postProcessingStarted == false && postProcessingDetector.find()) {
|
||||
postProcessingStarted = true;
|
||||
if (timeStamp == null) {
|
||||
timeStamp = new Date(process.getStartTime()).toInstant();
|
||||
}
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
timeStamp = Instant.now();
|
||||
process.setRenderDuration((int) phaseDuration.toSeconds());
|
||||
}
|
||||
|
||||
Matcher savingTimeDetector = savingPattern.matcher(line);
|
||||
if (savingTimeDetector.find()) {
|
||||
String savingTime = savingTimeDetector.group(1);
|
||||
if (savingTime != null) {
|
||||
savingTimeSeconds = (int) Duration.between(LocalTime.MIN, LocalTime.parse("00:" + savingTime)).toSeconds(); //add leading hours to comply with ISO time format
|
||||
}
|
||||
}
|
||||
|
||||
if (configuration.getMaxAllowedMemory() != -1 && getProcessRender().getMemoryUsed().get() > configuration.getMaxAllowedMemory()) {
|
||||
log.debug("Blocking render because process ram used (" + getProcessRender().getMemoryUsed().get() + "k) is over user setting (" + configuration
|
||||
.getMaxAllowedMemory() + "k)");
|
||||
process.finish();
|
||||
if (process.getRenderDuration() == -1) {
|
||||
if (timeStamp == null) {
|
||||
timeStamp = new Date(process.getStartTime()).toInstant();
|
||||
}
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
process.setRenderDuration((int) phaseDuration.toSeconds());
|
||||
}
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
if (disableViewportScript != null) {
|
||||
disableViewportScript.delete();
|
||||
}
|
||||
|
||||
// Once the process is finished (either finished successfully or with an error) move back to
|
||||
// base icon (isolated S with no progress bar)
|
||||
gui.updateTrayIcon(SHOW_BASE_ICON);
|
||||
|
||||
return Error.Type.RENDERER_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
updateSpeedSamplesRendered(line);
|
||||
updateRenderingStatus(line, progress);
|
||||
Type error = getRenderOutput().detectError(line);
|
||||
if (error != Error.Type.OK) {
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
if (disableViewportScript != null) {
|
||||
disableViewportScript.delete();
|
||||
}
|
||||
if (process.getRenderDuration() == -1) {
|
||||
if (timeStamp == null) {
|
||||
timeStamp = new Date(process.getStartTime()).toInstant();
|
||||
}
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
process.setRenderDuration((int) phaseDuration.toSeconds());
|
||||
}
|
||||
|
||||
// Put back base icon
|
||||
gui.updateTrayIcon(SHOW_BASE_ICON);
|
||||
process.kill();
|
||||
maybeCleanWorkingDir(error);
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
if (!event.isStarted() && (getProcessRender().getMemoryUsed().get() > 0 && scenePrepStarted || process.getRemainingDuration() > 0)) {
|
||||
event.doNotifyIsStarted();
|
||||
}
|
||||
}
|
||||
|
||||
if (timeStamp == null) {
|
||||
timeStamp = new Date(process.getStartTime()).toInstant();
|
||||
}
|
||||
|
||||
if (postProcessingStarted == false) {
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
process.setRenderDuration((int) phaseDuration.toSeconds());
|
||||
|
||||
//we need to subtract the time to save the frame to disk
|
||||
if (savingTimeSeconds > 0 && process.getRenderDuration() > 0) {
|
||||
process.setRenderDuration(Math.max(0, process.getRenderDuration() - savingTimeSeconds));
|
||||
}
|
||||
}
|
||||
else {
|
||||
phaseDuration = Duration.between(timeStamp, Instant.now());
|
||||
process.setPostProcessingDuration((int) phaseDuration.toSeconds());
|
||||
}
|
||||
input.close();
|
||||
|
||||
log.debug(String.format("render times: %n\tScene prep: %ds%n\tRendering: %ds%n\tPost: %ss%n\tTotal: %ds%n\tRendering/Total: %.03f%n",
|
||||
process.getScenePrepDuration(),
|
||||
process.getRenderDuration(),
|
||||
process.getPostProcessingDuration(),
|
||||
process.getDuration(),
|
||||
(Math.max(process.getRenderDuration(), 0) * 100.0) / process.getDuration()
|
||||
));
|
||||
}
|
||||
catch (IOException err1) { // for the input.readline
|
||||
// most likely The handle is invalid
|
||||
log.error("Job::render exception(B) (silent error) " + err1);
|
||||
}
|
||||
finally {
|
||||
memoryCheck.cancel();
|
||||
}
|
||||
|
||||
// Put back base icon
|
||||
gui.updateTrayIcon(SHOW_BASE_ICON);
|
||||
|
||||
log.debug("end of rendering");
|
||||
|
||||
}
|
||||
catch (Exception err) {
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
if (disableViewportScript != null) {
|
||||
disableViewportScript.delete();
|
||||
}
|
||||
StringWriter sw = new StringWriter();
|
||||
err.printStackTrace(new PrintWriter(sw));
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render exception(A) " + err + " stacktrace " + sw.toString());
|
||||
return Error.Type.FAILED_TO_EXECUTE;
|
||||
}
|
||||
|
||||
int exit_value = process.exitValue();
|
||||
process.finish();
|
||||
if (timerOfMaxRenderTime != null) {
|
||||
timerOfMaxRenderTime.cancel();
|
||||
}
|
||||
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
if (disableViewportScript != null) {
|
||||
disableViewportScript.delete();
|
||||
}
|
||||
// find the picture file
|
||||
final String filename_without_extension = getPrefixOutputImage() + getRenderSettings().getFrameNumber();
|
||||
|
||||
FilenameFilter textFilter = new FilenameFilter() {
|
||||
@Override public boolean accept(File dir, String name) {
|
||||
return name.startsWith(filename_without_extension);
|
||||
}
|
||||
};
|
||||
|
||||
File[] files = configuration.getWorkingDirectory().listFiles(textFilter);
|
||||
|
||||
if (getRenderState().isAskForRendererKill()) {
|
||||
log.debug("Job::render been asked to end render");
|
||||
|
||||
long duration = (new Date().getTime() - process.getStartTime()) / 1000; // in seconds
|
||||
if (configuration.getMaxRenderTime() > 0 && duration > configuration.getMaxRenderTime() && Integer.parseInt(this.getId()) >= 20) {
|
||||
log.debug("Render killed because process duration (" + duration + "s) is over user setting (" + configuration.getMaxRenderTime() + "s)");
|
||||
return Error.Type.RENDERER_KILLED_BY_USER_OVER_TIME;
|
||||
}
|
||||
|
||||
if (files.length != 0) {
|
||||
Arrays.stream(files).forEach( file -> new File(file.getAbsolutePath()).delete());
|
||||
}
|
||||
if (getRenderState().isServerBlock()) {
|
||||
return Error.Type.RENDERER_KILLED_BY_SERVER;
|
||||
}
|
||||
if (getRenderState().isUserBlock()) {
|
||||
return Error.Type.RENDERER_KILLED_BY_USER;
|
||||
}
|
||||
if (getRenderState().isIncompatibleProcessKill()) {
|
||||
return Error.Type.RENDERER_KILLED_BY_USER_INCOMPATIBLE_PROCESS;
|
||||
}
|
||||
return Error.Type.RENDERER_KILLED;
|
||||
}
|
||||
|
||||
if (files.length == 0) {
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render no picture file found (after finished render (filename_without_extension " + filename_without_extension + ")");
|
||||
|
||||
String basename = "";
|
||||
try {
|
||||
basename = getRenderSettings().getPath().substring(0, getRenderSettings().getPath().lastIndexOf('.'));
|
||||
}
|
||||
catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
File crash_file = new File(configuration.getWorkingDirectory() + File.separator + basename + ".crash.txt");
|
||||
if (crash_file.exists()) {
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render crash file found => the renderer crashed");
|
||||
crash_file.delete();
|
||||
return Error.Type.RENDERER_CRASHED;
|
||||
}
|
||||
|
||||
if (exit_value == 127 && process.getDuration() < 10) {
|
||||
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||
log.debug(logline);
|
||||
}
|
||||
log.error("Job::render renderer returned 127 and took " + process.getDuration() + "s, some libraries may be missing");
|
||||
return Error.Type.RENDERER_MISSING_LIBRARIES;
|
||||
}
|
||||
|
||||
return Error.Type.NOOUTPUTFILE;
|
||||
}
|
||||
else {
|
||||
if (files.length == 2) {
|
||||
Arrays.sort(files); //in case of an exr we end up with 2 images, the output as an exr and the preview as a jpg, we want to ensure the output comes first
|
||||
String path = files[1].getAbsolutePath();
|
||||
String extension = path.substring(path.lastIndexOf(".") + 1).toLowerCase();
|
||||
if ("jpg".equals(extension)) {
|
||||
getRenderOutput().setPreviewImagePath(files[1].getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
getRenderOutput().setFullImagePath(files[0].getAbsolutePath());
|
||||
getRenderOutput().setFullImageSize(new File(getRenderOutput().getFullImagePath()).length());
|
||||
log.debug(String.format("Job::render pictureFilename: %s, size: %d'", getRenderOutput().getFullImagePath(), getRenderOutput().getFullImageSize()));
|
||||
}
|
||||
|
||||
File scene_dir = new File(getSceneDirectory());
|
||||
long date_modification_scene_directory = (long) Utils.lastModificationTime(scene_dir);
|
||||
if (date_modification_scene_directory > process.getStartTime()) {
|
||||
scene_dir.delete();
|
||||
}
|
||||
|
||||
gui.status(String.format("Render time: %dmin%ds",
|
||||
process.getRenderDuration() / 60,
|
||||
process.getRenderDuration() % 60)
|
||||
);
|
||||
|
||||
return Error.Type.OK;
|
||||
}
|
||||
|
||||
private int computeRenderingProgress(String line, Pattern tilePattern, int currentProgress) {
|
||||
Matcher standardTileInfo = tilePattern.matcher(line);
|
||||
int newProgress = currentProgress;
|
||||
|
||||
if (standardTileInfo.find()) {
|
||||
int tileJustProcessed = Integer.parseInt(standardTileInfo.group(2));
|
||||
int totalTilesInJob = Integer.parseInt(standardTileInfo.group(3));
|
||||
|
||||
newProgress = Math.abs((tileJustProcessed * 100) / totalTilesInJob);
|
||||
}
|
||||
|
||||
// Only update the tray icon and the screen if percentage has changed
|
||||
if (newProgress != currentProgress) {
|
||||
gui.updateTrayIcon(newProgress);
|
||||
gui.status("Rendering", newProgress);
|
||||
}
|
||||
|
||||
return newProgress;
|
||||
}
|
||||
|
||||
private void updateSpeedSamplesRendered(String line) {
|
||||
// Looking for "Rendered 1281 samples in 66.319402 seconds"
|
||||
Pattern pattern = Pattern.compile("^Rendered (\\d+) samples in ([\\d.]+) seconds$");
|
||||
Matcher matcher = pattern.matcher(line);
|
||||
|
||||
if (matcher.find()) {
|
||||
int amount = Integer.parseInt(matcher.group(1));
|
||||
float duration = Float.parseFloat(matcher.group(2));
|
||||
|
||||
if (duration != 0 && amount != 0) {
|
||||
this.renderOutput.setSpeedSamplesRendered(amount / duration);
|
||||
}
|
||||
}
|
||||
|
||||
// should we use this, instead ???
|
||||
// "Average time per sample: 0.052112 seconds"
|
||||
}
|
||||
|
||||
private void updateRenderingStatus(String line, int progress) {
|
||||
if (getRenderSettings().getUpdateRenderingStatusMethod() == null || UPDATE_METHOD_BY_REMAINING_TIME.equals(getRenderSettings().getUpdateRenderingStatusMethod())) {
|
||||
String search_remaining = "remaining:";
|
||||
int index = line.toLowerCase().indexOf(search_remaining);
|
||||
if (index != -1) {
|
||||
String buf1 = line.substring(index + search_remaining.length());
|
||||
index = buf1.indexOf(" ");
|
||||
|
||||
if (index != -1) {
|
||||
String remaining_time = buf1.substring(0, index).trim();
|
||||
int last_index = remaining_time.lastIndexOf('.'); //format 00:00:00.00 (hr:min:sec)
|
||||
if (last_index > 0) {
|
||||
remaining_time = remaining_time.substring(0, last_index);
|
||||
}
|
||||
|
||||
try {
|
||||
DateFormat date_parse_minute = new SimpleDateFormat("m:s");
|
||||
DateFormat date_parse_hour = new SimpleDateFormat("h:m:s");
|
||||
DateFormat date_parse = date_parse_minute;
|
||||
if (remaining_time.split(":").length > 2) {
|
||||
date_parse = date_parse_hour;
|
||||
}
|
||||
date_parse.setTimeZone(TimeZone.getTimeZone("GMT"));
|
||||
Date date = date_parse.parse(remaining_time);
|
||||
gui.setRemainingTime(Utils.humanDuration(date));
|
||||
getProcessRender().setRemainingDuration((int) (date.getTime() / 1000));
|
||||
}
|
||||
catch (ParseException err) {
|
||||
log.error("Client::updateRenderingStatus ParseException " + err);
|
||||
}
|
||||
}
|
||||
}
|
||||
else { //extrapolate remaining time from time rendered & progress
|
||||
if (line.contains("Time") == true) {
|
||||
long timeRendered = new Date().getTime() - getProcessRender().getStartTime();
|
||||
|
||||
if (progress > 0 && timeRendered > 0) {
|
||||
long linearTimeEstimation = (long) ((100.0 / progress) * timeRendered);
|
||||
long timeRemaining = linearTimeEstimation - timeRendered;
|
||||
Date date = new Date(timeRemaining);
|
||||
|
||||
gui.setRemainingTime(Utils.humanDuration(date));
|
||||
getProcessRender().setRemainingDuration((int) (date.getTime() / 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (UPDATE_METHOD_BY_TILE.equals(getRenderSettings().getUpdateRenderingStatusMethod())) {
|
||||
String search = " Tile ";
|
||||
int index = line.lastIndexOf(search);
|
||||
if (index != -1) {
|
||||
String buf = line.substring(index + search.length());
|
||||
String[] parts = buf.split("/");
|
||||
if (parts.length == 2) {
|
||||
try {
|
||||
int current = Integer.parseInt(parts[0]);
|
||||
int total = Integer.parseInt(parts[1]);
|
||||
if (total != 0) {
|
||||
gui.status(String.format("Rendering %s %%", (int) (100.0 * current / total)));
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch (NumberFormatException e) {
|
||||
log.error("Exception 94: " + e);
|
||||
}
|
||||
}
|
||||
}
|
||||
gui.status("Rendering");
|
||||
}
|
||||
}
|
||||
|
||||
private void updateProcess() {
|
||||
getProcessRender().update();
|
||||
}
|
||||
|
||||
private void maybeCleanWorkingDir(Type error) {
|
||||
boolean cleanup = Type.COLOR_MANAGEMENT_ERROR == error
|
||||
|| Type.RENDERER_CRASHED_PYTHON_ERROR == error
|
||||
|| Type.ENGINE_NOT_AVAILABLE == error
|
||||
|| Type.DETECT_DEVICE_ERROR == error;
|
||||
if (cleanup) {
|
||||
directoryManager.cleanWorkingDirectory();
|
||||
}
|
||||
}
|
||||
|
||||
public static class renderStartedObservable extends Observable {
|
||||
|
||||
@Getter private boolean isStarted;
|
||||
|
||||
public renderStartedObservable(Observer observer) {
|
||||
super();
|
||||
addObserver(observer);
|
||||
}
|
||||
|
||||
public void doNotifyIsStarted() {
|
||||
setChanged();
|
||||
notifyObservers();
|
||||
isStarted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user