Fix CPU spikes

This commit is contained in:
harlekin
2021-12-21 10:08:59 +00:00
committed by Sheepit Renderfarm
parent 810253731d
commit 8aa56688ff
2 changed files with 38 additions and 39 deletions

View File

@@ -21,11 +21,9 @@ package com.sheepit.client;
import com.sheepit.client.Configuration.ComputeType; import com.sheepit.client.Configuration.ComputeType;
import com.sheepit.client.Error.Type; import com.sheepit.client.Error.Type;
import com.sheepit.client.hardware.cpu.CPU;
import com.sheepit.client.os.OS; import com.sheepit.client.os.OS;
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
import oshi.software.os.OSProcess;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
@@ -109,7 +107,7 @@ import java.util.regex.Pattern;
userBlockJob = false; userBlockJob = false;
serverBlockJob = false; serverBlockJob = false;
log = log_; log = log_;
render = new RenderProcess(); render = new RenderProcess(log_);
blenderShortVersion = null; blenderShortVersion = null;
blenderLongVersion = null; blenderLongVersion = null;
} }
@@ -133,25 +131,6 @@ import java.util.regex.Pattern;
frameNumber, sceneMD5, rendererMD5, id, outputImagePath, path, useGPU, name, extras, updateRenderingStatusMethod, render); frameNumber, sceneMD5, rendererMD5, id, outputImagePath, path, useGPU, name, extras, updateRenderingStatusMethod, render);
} }
private OSProcess getAsOSProcess(Process proc) {
return proc == null ? null : OS.operatingSystem.getProcess((int) proc.pid()); // Check for null otherwise proc.pid() will throw NullPointerEx
}
public long getUsedMemory() {
OSProcess osp = getAsOSProcess(getProcessRender().getProcess());
return osp != null ? osp.getResidentSetSize() / 1024 : 0;
}
public long getTotalUsedMemory() {
OSProcess osp = getAsOSProcess(getProcessRender().getProcess());
return osp != null ? osp.getVirtualSize() / 1024 : 0;
}
private int getThreadCount() {
OSProcess osp = getAsOSProcess(getProcessRender().getProcess());
return osp != null ? osp.getThreadCount() : 0;
}
public String getPrefixOutputImage() { public String getPrefixOutputImage() {
return id + "_"; return id + "_";
} }
@@ -305,13 +284,14 @@ import java.util.regex.Pattern;
process.setCoresUsed(configuration.getNbCores()); process.setCoresUsed(configuration.getNbCores());
process.start(); process.start();
getProcessRender().setProcess(os.exec(command, new_env)); getProcessRender().setProcess(os.exec(command, new_env));
getProcessRender().setOsProcess(OS.operatingSystem.getProcess((int) getProcessRender().getProcess().pid()));
BufferedReader input = new BufferedReader(new InputStreamReader(getProcessRender().getProcess().getInputStream())); BufferedReader input = new BufferedReader(new InputStreamReader(getProcessRender().getProcess().getInputStream()));
memoryCheck.scheduleAtFixedRate(new TimerTask() { memoryCheck.scheduleAtFixedRate(new TimerTask() {
@Override @Override
public void run() { public void run() {
updateRenderingMemoryPeak(); updateProcess();
} }
}, 0L, 1000L); }, 0L, 200L);
// Make initial test/power frames ignore the maximum render time in user configuration. Initial test frames have Job IDs below 20 // Make initial test/power frames ignore the maximum render time in user configuration. Initial test frames have Job IDs below 20
// so we just activate the user defined timeout when the scene is not one of the initial ones. // so we just activate the user defined timeout when the scene is not one of the initial ones.
@@ -357,8 +337,8 @@ import java.util.regex.Pattern;
} }
progress = computeRenderingProgress(line, tilePattern, progress); progress = computeRenderingProgress(line, tilePattern, progress);
if (configuration.getMaxAllowedMemory() != -1 && getUsedMemory() > configuration.getMaxAllowedMemory()) { if (configuration.getMaxAllowedMemory() != -1 && getProcessRender().getMemoryUsed().get() > configuration.getMaxAllowedMemory()) {
log.debug("Blocking render because process ram used (" + getUsedMemory() + "k) is over user setting (" + configuration log.debug("Blocking render because process ram used (" + getProcessRender().getMemoryUsed().get() + "k) is over user setting (" + configuration
.getMaxAllowedMemory() + "k)"); .getMaxAllowedMemory() + "k)");
OS.getOS().kill(process.getProcess()); OS.getOS().kill(process.getProcess());
process.finish(); process.finish();
@@ -386,7 +366,7 @@ import java.util.regex.Pattern;
return error; return error;
} }
if (!event.isStarted() && (getUsedMemory() > 0 || process.getRemainingDuration() > 0)) { if (!event.isStarted() && (getProcessRender().getMemoryUsed().get() > 0 || process.getRemainingDuration() > 0)) {
event.doNotifyIsStarted(); event.doNotifyIsStarted();
} }
} }
@@ -600,15 +580,8 @@ import java.util.regex.Pattern;
} }
} }
private void updateRenderingMemoryPeak() { private void updateProcess() {
Process proc = getProcessRender().getProcess(); getProcessRender().update();
if (proc != null && getAsOSProcess(proc) != null){
long mem = getUsedMemory();
getProcessRender().setMemoryUsed(mem);
if (getProcessRender().getPeakMemoryUsed() < mem) {
getProcessRender().setPeakMemoryUsed(mem);
}
}
} }
private Type detectError(String line) { private Type detectError(String line) {

View File

@@ -22,24 +22,49 @@ package com.sheepit.client;
import lombok.Data; import lombok.Data;
import java.util.Date; import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;
import oshi.software.os.OSProcess;
@Data public class RenderProcess { @Data public class RenderProcess {
private long startTime; private long startTime;
private long endTime; private long endTime;
private int remainingDuration; // in seconds private int remainingDuration; // in seconds
private long memoryUsed; // in kB private AtomicLong memoryUsed; // in kB
private long peakMemoryUsed; // in kB private long peakMemoryUsed; // in kB
private int coresUsed; private int coresUsed;
private Process process; private Process process;
private OSProcess osProcess;
private Log log;
public RenderProcess() { public RenderProcess(Log _log) {
process = null; process = null;
osProcess = null;
startTime = -1; startTime = -1;
endTime = -1; endTime = -1;
memoryUsed = 0; memoryUsed = new AtomicLong(0);
peakMemoryUsed = 0; peakMemoryUsed = 0;
coresUsed = 0; coresUsed = 0;
remainingDuration = 0; remainingDuration = 0;
log = _log;
}
public void update() {
OSProcess osp = osProcess; // Shallow copy to try to not run into a race condition via being nulled
try {
if (osp != null && osp.updateAttributes()){ // We enter if updateAttributes() was successful
long mem = osp.getResidentSetSize() / 1024; // Avoid multiple ram usage calls, because again, they might differ
if (mem != 0){
memoryUsed.set(mem);
if (peakMemoryUsed < mem) {
peakMemoryUsed = mem;
}
}
}
} catch (NullPointerException ex) { // We are racing the system itself, we can't avoid catching NPE's
log.debug("RenderProcess::Handled process becoming unavailable mid-update");
osProcess = null;
memoryUsed.set(0);
}
} }
/** /**
@@ -57,6 +82,7 @@ import java.util.Date;
public void finish() { public void finish() {
endTime = new Date().getTime(); endTime = new Date().getTime();
osProcess = null;
process = null; process = null;
} }