Feat: change the architecture of the 'sheepit network', instead of storing every frame,mp4,zip on a single server, use multiple servers(shepherds)

This commit is contained in:
Laurent Clouet
2020-04-14 17:35:54 +02:00
parent 08fe55564c
commit 57bc27bdcf
6 changed files with 18 additions and 13 deletions

View File

@@ -34,7 +34,6 @@ A path is provided for error, job request, job validation, download needed file,
The maximum duration between two heartbeats in seconds is given by the attribute "max-period". The maximum duration between two heartbeats in seconds is given by the attribute "max-period".
<?xml version="1.0" encoding="utf-8" ?> <?xml version="1.0" encoding="utf-8" ?>
<config status="0"> <config status="0">
<request type="validate-job" path="/server/send_frame.php" />
<request type="request-job" path="/server/request_job.php" /> <request type="request-job" path="/server/request_job.php" />
<request type="download-archive" path="/server/download.php" /> <request type="download-archive" path="/server/download.php" />
<request type="error" path="/server/error.php" /> <request type="error" path="/server/error.php" />
@@ -123,7 +122,7 @@ if fileformat != 'BMP' and fileformat != 'PNG' and fileformat != 'JPEG' and file
=== Job validation === === Job validation ===
Url: use the request type "validate-job" from the configuration answer. Url: use the url url of job's node for 'validationurl'.
Parameter as GET or POST: Parameter as GET or POST:
* job: Job ID * job: Job ID
* frame: Job's frame number * frame: Job's frame number

View File

@@ -741,13 +741,9 @@ public class Client {
} }
protected Error.Type confirmJob(Job ajob) { protected Error.Type confirmJob(Job ajob) {
String extras_config = ""; String url_real = String.format("%s&rendertime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getDuration(), ajob.getProcessRender().getMemoryUsed());
RenderProcess process = ajob.getProcessRender(); this.log.debug("Client::confirmeJob url " + url_real);
if (process != null && process.getCoresUsed() > 0) { this.log.debug("path frame " + ajob.getOutputImagePath());
extras_config = "&cores=" + process.getCoresUsed();
}
String url_real = String.format("%s?job=%s&frame=%s&rendertime=%d&memoryused=%s&extras=%s%s", this.server.getPage("validate-job"), ajob.getId(), ajob.getFrameNumber(), ajob.getProcessRender().getDuration(), ajob.getProcessRender().getMemoryUsed(), ajob.getExtras(), extras_config);
this.isValidatingJob = true; this.isValidatingJob = true;
int nb_try = 1; int nb_try = 1;

View File

@@ -275,7 +275,7 @@ public class Configuration {
InputStream versionStream = Client.class.getResourceAsStream(versionPath); InputStream versionStream = Client.class.getResourceAsStream(versionPath);
if (versionStream == null) { if (versionStream == null) {
System.err.println("Configuration::getJarVersion Failed to get version file"); System.err.println("Configuration::getJarVersion Failed to get version file");
return "5.0.0"; return "6.0.0";
} }
try { try {
@@ -287,7 +287,7 @@ public class Configuration {
} }
catch (IOException ex) { catch (IOException ex) {
System.err.println("Configuration::getJarVersion error while reading manifest file (" + versionPath + "): " + ex.getMessage()); System.err.println("Configuration::getJarVersion error while reading manifest file (" + versionPath + "): " + ex.getMessage());
return "5.0.0"; return "6.0.0";
} }
} }

View File

@@ -66,6 +66,7 @@ public class Job {
private String outputImagePath; private String outputImagePath;
private String path; // path inside of the archive private String path; // path inside of the archive
private String rendererCommand; private String rendererCommand;
private String validationUrl;
private String script; private String script;
private boolean useGPU; private boolean useGPU;
private String name; private String name;
@@ -81,13 +82,14 @@ public class Job {
private Configuration configuration; private Configuration configuration;
private Log log; private Log log;
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String script_, String sceneMd5_, String rendererMd5_, String name_, String password_, String extras_, boolean synchronous_upload_, String update_method_) { public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_, String script_, String sceneMd5_, String rendererMd5_, String name_, String password_, String extras_, boolean synchronous_upload_, String update_method_) {
configuration = config_; configuration = config_;
id = id_; id = id_;
frameNumber = frame_; frameNumber = frame_;
path = path_; path = path_;
useGPU = use_gpu; useGPU = use_gpu;
rendererCommand = command_; rendererCommand = command_;
validationUrl = validationUrl_;
sceneMD5 = sceneMd5_; sceneMD5 = sceneMd5_;
rendererMD5 = rendererMd5_; rendererMD5 = rendererMd5_;
name = name_; name = name_;

View File

@@ -38,6 +38,7 @@ import java.net.CookieManager;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.NoRouteToHostException; import java.net.NoRouteToHostException;
import java.net.URLDecoder;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.net.URL; import java.net.URL;
import java.net.URLEncoder; import java.net.URLEncoder;
@@ -339,6 +340,8 @@ public class Server extends Thread implements HostnameVerifier, X509TrustManager
script += jobData.getRenderTask().getScript(); script += jobData.getRenderTask().getScript();
String validationUrl = URLDecoder.decode(jobData.getRenderTask().getValidationUrl());
Job a_job = new Job( Job a_job = new Job(
this.user_config, this.user_config,
this.client.getGui(), this.client.getGui(),
@@ -348,6 +351,7 @@ public class Server extends Thread implements HostnameVerifier, X509TrustManager
jobData.getRenderTask().getPath().replace("/", File.separator), jobData.getRenderTask().getPath().replace("/", File.separator),
jobData.getRenderTask().getUseGpu() == 1, jobData.getRenderTask().getUseGpu() == 1,
jobData.getRenderTask().getRendererInfos().getCommandline(), jobData.getRenderTask().getRendererInfos().getCommandline(),
validationUrl,
script, script,
jobData.getRenderTask().getArchive_md5(), jobData.getRenderTask().getArchive_md5(),
jobData.getRenderTask().getRendererInfos().getMd5(), jobData.getRenderTask().getRendererInfos().getMd5(),

View File

@@ -38,6 +38,10 @@ public class RenderTask {
@Getter @Getter
private String extras; private String extras;
@Attribute(name = "validation_url")
@Getter
private String validationUrl;
@Attribute(name = "name") @Attribute(name = "name")
@Getter @Getter
private String name; private String name;