Cleanup: move Job render call to Job class
This commit is contained in:
Laurent Clouet
@@ -19,25 +19,14 @@
|
||||
|
||||
package com.sheepit.client;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TimeZone;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
@@ -47,13 +36,9 @@ import com.sheepit.client.exception.FermeException;
|
||||
import com.sheepit.client.exception.FermeExceptionNoRightToRender;
|
||||
import com.sheepit.client.exception.FermeExceptionNoSession;
|
||||
import com.sheepit.client.exception.FermeExceptionSessionDisabled;
|
||||
import com.sheepit.client.hardware.gpu.GPUDevice;
|
||||
import com.sheepit.client.os.OS;
|
||||
|
||||
public class Client {
|
||||
public static final String UPDATE_METHOD_BY_REMAINING_TIME = "remainingtime";
|
||||
public static final String UPDATE_METHOD_BLENDER_INTERNAL_BY_PART = "blenderinternal";
|
||||
|
||||
private Gui gui;
|
||||
private Server server;
|
||||
private Configuration config;
|
||||
@@ -102,6 +87,10 @@ public class Client {
|
||||
return this.server;
|
||||
}
|
||||
|
||||
public Log getLog() {
|
||||
return this.log;
|
||||
}
|
||||
|
||||
public int run() {
|
||||
if (this.config.checkOSisSupported() == false) {
|
||||
this.gui.error(Error.humanString(Error.Type.OS_NOT_SUPPORTED));
|
||||
@@ -512,7 +501,7 @@ public class Client {
|
||||
return Error.Type.MISSING_RENDER;
|
||||
}
|
||||
|
||||
Error.Type err = this.runRenderer(ajob);
|
||||
Error.Type err = ajob.render();
|
||||
if (err != Error.Type.OK) {
|
||||
this.log.error("Client::work problem with runRenderer (ret " + err + ")");
|
||||
return err;
|
||||
@@ -521,188 +510,6 @@ public class Client {
|
||||
return Error.Type.OK;
|
||||
}
|
||||
|
||||
protected Error.Type runRenderer(Job ajob) {
|
||||
this.gui.status("Rendering");
|
||||
RenderProcess process = ajob.getProcessRender();
|
||||
String core_script = "import bpy\n" + "bpy.context.user_preferences.system.compute_device_type = \"%s\"\n" + "bpy.context.scene.cycles.device = \"%s\"\n" + "bpy.context.user_preferences.system.compute_device = \"%s\"\n";
|
||||
if (ajob.getUseGPU() && this.config.getGPUDevice() != null) {
|
||||
core_script = String.format(core_script, "CUDA", "GPU", this.config.getGPUDevice().getCudaName());
|
||||
}
|
||||
else {
|
||||
core_script = String.format(core_script, "NONE", "CPU", "CPU");
|
||||
}
|
||||
core_script += String.format("bpy.context.scene.render.tile_x = %1$d\nbpy.context.scene.render.tile_y = %1$d\n", this.getTileSize(ajob));
|
||||
File script_file = null;
|
||||
String command1[] = ajob.getRenderCommand().split(" ");
|
||||
int size_command = command1.length + 2; // + 2 for script
|
||||
|
||||
if (this.config.getNbCores() > 0) { // user has specified something
|
||||
size_command += 2;
|
||||
}
|
||||
|
||||
List<String> command = new ArrayList<String>(size_command);
|
||||
|
||||
Map<String, String> new_env = new HashMap<String, String>();
|
||||
|
||||
new_env.put("BLENDER_USER_CONFIG", this.config.workingDirectory.getAbsolutePath().replace("\\", "\\\\"));
|
||||
|
||||
for (String arg : command1) {
|
||||
switch (arg) {
|
||||
case ".c":
|
||||
command.add(ajob.getScenePath());
|
||||
command.add("-P");
|
||||
|
||||
try {
|
||||
script_file = File.createTempFile("script_", "", this.config.workingDirectory);
|
||||
File file = new File(script_file.getAbsolutePath());
|
||||
FileWriter txt;
|
||||
txt = new FileWriter(file);
|
||||
|
||||
PrintWriter out = new PrintWriter(txt);
|
||||
out.write(ajob.getScript());
|
||||
out.write("\n");
|
||||
out.write(core_script); // GPU part
|
||||
out.write("\n"); // GPU part
|
||||
out.close();
|
||||
|
||||
command.add(script_file.getAbsolutePath());
|
||||
}
|
||||
catch (IOException e) {
|
||||
StringWriter sw = new StringWriter();
|
||||
e.printStackTrace(new PrintWriter(sw));
|
||||
this.log.error("Client:runRenderer exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
||||
return Error.Type.UNKNOWN;
|
||||
}
|
||||
script_file.deleteOnExit();
|
||||
break;
|
||||
case ".e":
|
||||
command.add(ajob.getRendererPath());
|
||||
// the number of cores has to be put after the binary and before the scene arg
|
||||
if (this.config.getNbCores() > 0) {
|
||||
command.add("-t");
|
||||
command.add(Integer.toString(this.config.getNbCores()));
|
||||
}
|
||||
break;
|
||||
case ".o":
|
||||
command.add(this.config.workingDirectory.getAbsolutePath() + File.separator + ajob.getPrefixOutputImage());
|
||||
break;
|
||||
case ".f":
|
||||
command.add(ajob.getFrameNumber());
|
||||
break;
|
||||
default:
|
||||
command.add(arg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
String line;
|
||||
this.log.debug(command.toString());
|
||||
OS os = OS.getOS();
|
||||
process.start();
|
||||
ajob.getProcessRender().setProcess(os.exec(command, new_env));
|
||||
BufferedReader input = new BufferedReader(new InputStreamReader(ajob.getProcessRender().getProcess().getInputStream()));
|
||||
|
||||
long last_update_status = 0;
|
||||
this.log.debug("renderer output");
|
||||
try {
|
||||
while ((line = input.readLine()) != null) {
|
||||
this.updateRenderingMemoryPeak(line, ajob);
|
||||
|
||||
this.log.debug(line);
|
||||
if ((new Date().getTime() - last_update_status) > 2000) { // only call the update every two seconds
|
||||
this.updateRenderingStatus(line, ajob);
|
||||
last_update_status = new Date().getTime();
|
||||
}
|
||||
Type error = this.detectError(line, ajob);
|
||||
if (error != Error.Type.OK) {
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
return error;
|
||||
}
|
||||
}
|
||||
input.close();
|
||||
}
|
||||
catch (IOException err1) { // for the input.readline
|
||||
// most likely The handle is invalid
|
||||
this.log.error("Client:runRenderer exception(B) (silent error) " + err1);
|
||||
}
|
||||
this.log.debug("end of rendering");
|
||||
}
|
||||
catch (Exception err) {
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
StringWriter sw = new StringWriter();
|
||||
err.printStackTrace(new PrintWriter(sw));
|
||||
this.log.error("Client:runRenderer exception(A) " + err + " stacktrace " + sw.toString());
|
||||
return Error.Type.FAILED_TO_EXECUTE;
|
||||
}
|
||||
|
||||
int exit_value = process.exitValue();
|
||||
process.finish();
|
||||
|
||||
if (script_file != null) {
|
||||
script_file.delete();
|
||||
}
|
||||
|
||||
// find the picture file
|
||||
final String filename_without_extension = ajob.getPrefixOutputImage() + ajob.getFrameNumber();
|
||||
|
||||
FilenameFilter textFilter = new FilenameFilter() {
|
||||
public boolean accept(File dir, String name) {
|
||||
return name.startsWith(filename_without_extension);
|
||||
}
|
||||
};
|
||||
|
||||
File[] files = this.config.workingDirectory.listFiles(textFilter);
|
||||
|
||||
if (files.length == 0) {
|
||||
this.log.error("Client::runRenderer no picture file found (after finished render (filename_without_extension " + filename_without_extension + ")");
|
||||
|
||||
if (ajob.getAskForRendererKill()) {
|
||||
this.log.debug("Client::runRenderer renderer didn't generate any frame but died due to a kill request");
|
||||
return Error.Type.RENDERER_KILLED;
|
||||
}
|
||||
|
||||
String basename = "";
|
||||
try {
|
||||
basename = ajob.getPath().substring(0, ajob.getPath().lastIndexOf('.'));
|
||||
}
|
||||
catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
File crash_file = new File(this.config.workingDirectory + File.separator + basename + ".crash.txt");
|
||||
if (crash_file.exists()) {
|
||||
this.log.error("Client::runRenderer crash file found => the renderer crashed");
|
||||
crash_file.delete();
|
||||
return Error.Type.RENDERER_CRASHED;
|
||||
}
|
||||
|
||||
if (exit_value == 127 && process.getDuration() < 10) {
|
||||
this.log.error("Client::runRenderer renderer returned 127 and took " + process.getDuration() + "s, some libraries may be missing");
|
||||
return Error.Type.RENDERER_MISSING_LIBRARIES;
|
||||
}
|
||||
|
||||
return Error.Type.NOOUTPUTFILE;
|
||||
}
|
||||
else {
|
||||
ajob.setOutputImagePath(files[0].getAbsolutePath());
|
||||
this.log.debug("Client::runRenderer pictureFilename: '" + ajob.getOutputImagePath() + "'");
|
||||
}
|
||||
|
||||
File scene_dir = new File(ajob.getSceneDirectory());
|
||||
long date_modification_scene_directory = (long) Utils.lastModificationTime(scene_dir);
|
||||
if (date_modification_scene_directory > process.getStartTime()) {
|
||||
scene_dir.delete();
|
||||
}
|
||||
|
||||
this.gui.status(String.format("Frame rendered in %dmin%ds", process.getDuration() / 60, process.getDuration() % 60));
|
||||
|
||||
return Error.Type.OK;
|
||||
}
|
||||
|
||||
protected int downloadSceneFile(Job ajob_) {
|
||||
this.gui.status("Downloading project");
|
||||
return this.downloadFile(ajob_, ajob_.getSceneArchivePath(), ajob_.getSceneMD5(), String.format("%s?type=job&job=%s&revision=%s", this.server.getPage("download-archive"), ajob_.getId(), ajob_.getRevision()), "Downloading project %s %%");
|
||||
@@ -875,144 +682,4 @@ public class Client {
|
||||
}
|
||||
return (concurrent_job >= this.config.maxUploadingJob());
|
||||
}
|
||||
|
||||
protected void updateRenderingStatus(String line, Job ajob) {
|
||||
if (ajob.getUpdateRenderingStatusMethod() != null && ajob.getUpdateRenderingStatusMethod().equals(Client.UPDATE_METHOD_BLENDER_INTERNAL_BY_PART)) {
|
||||
String search = " Part ";
|
||||
int index = line.lastIndexOf(search);
|
||||
if (index != -1) {
|
||||
String buf = line.substring(index + search.length());
|
||||
String[] parts = buf.split("-");
|
||||
if (parts != null && parts.length == 2) {
|
||||
try {
|
||||
int current = Integer.parseInt(parts[0]);
|
||||
int total = Integer.parseInt(parts[1]);
|
||||
if (total != 0) {
|
||||
this.gui.status(String.format("Rendering %s %%", (int) (100.0 * current / total)));
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch (NumberFormatException e) {
|
||||
System.out.println("Exception 92: " + e);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.gui.status("Rendering");
|
||||
}
|
||||
else if (ajob.getUpdateRenderingStatusMethod() == null || ajob.getUpdateRenderingStatusMethod().equals(Client.UPDATE_METHOD_BY_REMAINING_TIME)) {
|
||||
String search_remaining = "remaining:";
|
||||
int index = line.toLowerCase().indexOf(search_remaining);
|
||||
if (index != -1) {
|
||||
String buf1 = line.substring(index + search_remaining.length());
|
||||
index = buf1.indexOf(" ");
|
||||
|
||||
if (index != -1) {
|
||||
String remaining_time = buf1.substring(0, index).trim();
|
||||
int last_index = remaining_time.lastIndexOf('.'); //format 00:00:00.00 (hr:min:sec)
|
||||
if (last_index > 0) {
|
||||
remaining_time = remaining_time.substring(0, last_index);
|
||||
}
|
||||
|
||||
try {
|
||||
DateFormat date_parse_minute = new SimpleDateFormat("m:s");
|
||||
DateFormat date_parse_hour = new SimpleDateFormat("h:m:s");
|
||||
DateFormat date_parse = date_parse_minute;
|
||||
if (remaining_time.split(":").length > 2) {
|
||||
date_parse = date_parse_hour;
|
||||
}
|
||||
date_parse.setTimeZone(TimeZone.getTimeZone("GMT"));
|
||||
Date date = date_parse.parse(remaining_time);
|
||||
this.gui.status(String.format("Rendering (remaining %s)", Utils.humanDuration(date)));
|
||||
ajob.getProcessRender().setRemainingDuration((int) (date.getTime() / 1000));
|
||||
}
|
||||
catch (ParseException err) {
|
||||
this.log.error("Client::updateRenderingStatus ParseException " + err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void updateRenderingMemoryPeak(String line, Job ajob) {
|
||||
String[] elements = line.toLowerCase().split("(peak)");
|
||||
|
||||
for (String element : elements) {
|
||||
if (element.isEmpty() == false && element.charAt(0) == ' ') {
|
||||
int end = element.indexOf(')');
|
||||
if (end > 0) {
|
||||
long mem = Utils.parseNumber(element.substring(1, end).trim());
|
||||
if (mem > ajob.getProcessRender().getMemoryUsed()) {
|
||||
ajob.getProcessRender().setMemoryUsed(mem);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (element.isEmpty() == false && element.charAt(0) == ':') {
|
||||
int end = element.indexOf('|');
|
||||
if (end > 0) {
|
||||
long mem = Utils.parseNumber(element.substring(1, end).trim());
|
||||
if (mem > ajob.getProcessRender().getMemoryUsed()) {
|
||||
ajob.getProcessRender().setMemoryUsed(mem);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Type detectError(String line, Job ajob) {
|
||||
|
||||
if (line.indexOf("CUDA error: Out of memory") != -1) {
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.26M | Scene, RenderLayer | Updating Device | Writing constant memory
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.26M | Scene, RenderLayer | Path Tracing Tile 0/135, Sample 0/200
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.82M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 1/135, Sample 0/200
|
||||
// CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
|
||||
// Refer to the Cycles GPU rendering documentation for possible solutions:
|
||||
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:09:26.57 | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 1/135, Sample 200/200
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.06 | Mem:470.50M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 134/135, Sample 0/200
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.03 | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 134/135, Sample 200/200
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Remaining:00:00.03 | Mem:470.50M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 135/135, Sample 0/200
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Path Tracing Tile 135/135, Sample 200/200
|
||||
// Error: CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
|
||||
// Fra:151 Mem:405.91M (0.00M, Peak 633.81M) | Mem:470.26M, Peak:470.82M | Scene, RenderLayer | Cancel | CUDA error: Out of memory in cuLaunchKernel(cuPathTrace, xblocks , yblocks, 1, xthreads, ythreads, 1, 0, 0, args, 0)
|
||||
// Fra:151 Mem:405.89M (0.00M, Peak 633.81M) Sce: Scene Ve:0 Fa:0 La:0
|
||||
// Saved: /tmp/xx/26885_0151.png Time: 00:04.67 (Saving: 00:00.22)
|
||||
// Blender quit
|
||||
return Type.RENDERER_OUT_OF_VIDEO_MEMORY;
|
||||
}
|
||||
else if (line.indexOf("CUDA device supported only with compute capability") != -1) {
|
||||
// found bundled python: /tmp/xx/2.73/python
|
||||
// read blend: /tmp/xx/compute-method.blend
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Sun
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Plane
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Cube
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Synchronizing object | Camera
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Initializing
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Loading render kernels (may take a few minutes the first time)
|
||||
// CUDA device supported only with compute capability 2.0 or up, found 1.2.
|
||||
// Refer to the Cycles GPU rendering documentation for possible solutions:
|
||||
// http://www.blender.org/manual/render/cycles/gpu_rendering.html
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Error | CUDA device supported only with compute capability 2.0 or up, found 1.2.
|
||||
// Error: CUDA device supported only with compute capability 2.0 or up, found 1.2.
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Waiting for render to start
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) | Mem:0.00M, Peak:0.00M | Scene, RenderLayer | Cancel | CUDA device supported only with compute capability 2.0 or up, found 1.2.
|
||||
// Fra:340 Mem:7.64M (0.00M, Peak 8.23M) Sce: Scene Ve:0 Fa:0 La:0
|
||||
// Saved: /tmp/xx/0_0340.png Time: 00:00.12 (Saving: 00:00.03)
|
||||
// Blender quit
|
||||
return Type.GPU_NOT_SUPPORTED;
|
||||
}
|
||||
return Type.OK;
|
||||
}
|
||||
|
||||
protected int getTileSize(Job ajob) {
|
||||
int size = 32; // CPU
|
||||
GPUDevice gpu = this.config.getGPUDevice();
|
||||
if (ajob.getUseGPU() && this.config.getGPUDevice() != null) {
|
||||
// GPU
|
||||
// if the vram is lower than 1G reduce the size of tile to avoid black output
|
||||
size = (gpu.getMemory() > 1073741824L) ? 256 : 128;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user