feature: mirror speedtest

protocol.txt

@@ -31,21 +31,31 @@ where X:
     * something else => Unknown error.
 
 Answer with no error:
-A status of "0" to specify everything is okay, plus a list of URL paths for request, validation job, etc.
+A status of "0" to specify everything is okay, publickey contains the public key for login, generated by the server, plus a list of URL paths for request, validation job, etc.
-A path is provided for error, job request, job validation, download needed file, heartbeat (keepmealive), logout, thumbnail of last frame rendered and how many credits have been earned so far.
+A path is provided for error, job request, job validation, download needed file, heartbeat (keepmealive), logout and speedtest results.
+Also a speedtest element containing a url for each mirror leading to a payload for the ensuing speedtest.
 The maximum duration between two heartbeats in seconds is given by the attribute "max-period".
+
 <?xml version="1.0" encoding="utf-8" ?>
-<config status="0">
+<config status="0" publickey="a public key">
     <request type="request-job" path="/server/request_job.php" />
-    <request type="download-archive" path="/server/download.php" />
+    <request type="download-archive" path="/server/archive.php" />
     <request type="error" path="/server/error.php" />
     <request type="keepmealive" path="/server/keepmealive.php" max-period="1440" />
-    <request type="logout" path="/account.php?mode=logout&worker=1" />
+    <request type="logout" path="/account.php?mode=logout&amp;worker=1" />
-    <request type="last-render-frame" path="/ajax.php?action=webclient_get_last_render_frame_ui&type=raw"/>
+    <request type="speedtest-answer" path="/server/speedtest.php" />
-    <request type="credits-earned" path="/ajax.php?action=credits_earned_on_session"/>
+    <speedtest>
+        <target url="https://static-frankfurt3-de.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-murcia-es.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-nuremberg-de.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-roubaix-fr.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-sg.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-tx3-usa.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-ut-usa.sheepit-renderfarm.com/scene/speedtest.zip" />
+        <target url="https://static-va-usa.sheepit-renderfarm.com/scene/speedtest.zip" />
+    </speedtest>
 </config>
 
-
 === Session end ===
 
 Url: use the request type "logout" from the configuration answer.
@@ -80,7 +90,11 @@ On error: an 404 http code
 Url: use the request type "request-job" from the configuration answer.
 Parameter as GET or POST:
     * computemethod: What compute types are available on this machine. 0 for CPU or GPU, 1 for CPU only, 2 for GPU only.
+    * network_dl: Download speed in bytes/second
+    * network_up: Upload speed in bytes/second
     * cpu_cores: Number of cores currently available for rendering (optional).
+    * ram_max: Maximum memory allowed for renderer (in kilobytes).
+    * rendertime_max: Maximum allowed render time in seconds, 0 means no time limit
     * gpu_type: GPU's type, usually CUDA or OPENCL
     * gpu_model: Model name of the GPU available for rendering
     * gpu_ram: GPU memory size (in bytes)
@@ -102,25 +116,43 @@ where X:
 Answer with no error:
 <?xml version="1.0" encoding="utf-8" ?>
 <jobrequest status="0">
-    <frames remaining="1187" />
+	<stats credits_session="0" credits_total="6318899" frame_remaining="36830" waiting_project="42" connected_machine="773" renderable_project="0" />
-    <job id="1" use_gpu="1" archive_md5="11d046f9912267a29f99a731c7e4e3b0" path="compute_method.blend" frame="0340" synchronous_upload="1" name="human name" password="some_passowrd">
+	<job id="1" use_gpu="1" archive_md5="db26b54689516484633b7d4855fb1567" path="compute-method.blend" frame="0340" synchronous_upload="1" extras="" validation_url="https%3A%2F%2Fsheepit-renderfarm.com%2Fserver%2Fsend_frame.php%3Fjob%3D1%26frame%3D0340%26extras%3D" name="human readable name" password="some password">
-        <renderer md5="ceda00890578762c6fac96f1a13f671a" commandline=".e --factory-startup --disable-autoexec -b .c -o .o -f .f -x 1" update_method="remainingtime"/>
+		<renderer md5="05234503229a4660e428e8d227746d8d" commandline=".e --factory-startup --disable-autoexec -noaudio -b .c --engine CYCLES -o .o -f .f -x 1" update_method="remainingtime"/>
-        <script>import bpy
+		<script>
-# disable the GPU for Cycles
+			<!-- a server generated python script configuring the render with the right settings for the farm -->
-bpy.context.user_preferences.system.compute_device_type = "NONE"
+		</script>
-
+	</job>
-# if it's a movie clip, switch to png
-fileformat = bpy.context.scene.render.image_settings.file_format
-if fileformat != 'BMP' and fileformat != 'PNG' and fileformat != 'JPEG' and fileformat != 'TARGA' and fileformat != 'TARGA_RAW' :
-    bpy.context.scene.render.image_settings.file_format = 'PNG'
-    #bpy.context.scene.render.file_extension = '.png'
-    bpy.context.scene.render.filepath = ''
-
-        </script>
-    </job>
 </jobrequest>
 
+stats => Some statistics about the session and the farm
+	* where
+		* credits_session => The points earned in this session
+		* credits_total => Represents the total amount of points of this account
+		* frame_remaining => How many frames are left to be rendered on the farm in total
+		* waiting_project => How many projects are in the render queue
+		* connected_machines => How many machines are connected to the farm right now
+		* renderable_projects => How many projects are available to be rendered by this session/machine
+job => an element containing information about the new job
+	* where
+		* job id => The job id
+		* use_gpu => A flag selecting the render device (CPU/GPU) according to the session settings and the project (for example when a CPU+GPU session receives a CPU only project), 0 means false, 1 true
+		* archive_md5 => The MD5 checksum of the project zip, used to verify integrity after download
+		* path => The path to the blend file within the archive
+		* frame => The frame number to be rendered
+		* synchronous_upload => A flag indicating whether the job result can be queued up for uploading (0) or should be sent back immediately (1, only for the test frames)
+		* extras => Additional information, like whether the frame is a tile
+		* validation_url => The URL to send the job validation to
+		* name => The project name that will be displayed in the ui
+		* password => The archive password
 
+		renderer => An element containing information about the blender binary used for the job
+			* where
+				* md5 => The MD5 checksum of the binary archive, used to validate file integrity after download
+				* commandline => Launch arguments for the blender executable
+				* update_method => Whether to keep track of progress by remaining time (remainingtime) or render tile (by_tile)
+
+		script => A server generated python script setting blender up with the right farm-related settings for the project
 
 === Job validation ===
 
@@ -137,7 +169,8 @@ Parameter as form-urlencoded:
     * file: the frame to send   
 
 Answer in case of error:
-<jobvalidate status="X" />
+<?xml version="1.0" encoding="utf-8"?>
+<jobvalidate status="X"/>
 where X:
     * 0 => No error
     * 300 => Missing parameter in request.
@@ -168,18 +201,42 @@ where X:
 
 Url: use the request type "error" from the configuration answer.
 Parameter as GET or POST:
+	* type: X
+		where X:
+			* 1 => The client received a wrong configuration
+			* 2 => Authentication with www failed
+			* 3 => Client version too old
+			* 4 => The session got disabled
+			* 5 => Blender binary not available
+			* 6 => Blender binary missing
+			* 7 => Couldn't find a scene in the blend file
+			* 8 => Rendering produced no output file
+			* 9 => File download failed
+			* 10 => Can not create directory
+			* 11 => Network issue
+			* 12 => Renderer crashed
+			* 13 => Renderer failed due to running out of VRAM
+			* 14 => Render process got killed by OS
+			* 15 => Renderer is missing libraries
+			* 16 => Process execution failed
+			* 17 => OS not supported
+			* 18 => CPU not supported
+			* 19 => GPU not supported
+			* 20 => Renderer got killed by user
+			* 21 => Renderer failed due to running out of RAM
+			* 22 => Renderer got killed by server
+			* 23 => Renderer got killed due to exceeding the user time limit
+			* 24 => Renderer crashed with a python error
+			* 25 => Job validation failed
+			* 26 => Final image is too large
+			* 27 => Render engine not available
+			* 99 => Unknown error
     * job: job ID
     * frame: job's frame number
     * extras: job's extra data
-    * rendertime (optional): job's frame number
+    * render_time (optional): job's frame number
     * memoryused (optional): max memory used for the render (in kilo bytes)
 Parameter as form-urlencoded:
     * file: the error log to send   
 
 
-Answer:
-<?xml version="1.0" encoding="utf-8" ?>
-<error status="X" />
-where X:
-    * 0 => No error
-

src/com/sheepit/client/Server.java

@@ -35,7 +35,11 @@ import java.util.Date;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
+import com.sheepit.client.datamodel.SpeedTestTarget;
+import com.sheepit.client.datamodel.SpeedTestResult;
+import com.sheepit.client.datamodel.SpeedTestTargetResult;
 import lombok.Getter;
 import org.simpleframework.xml.core.Persister;
 
@@ -74,6 +78,8 @@ import com.sheepit.client.os.OS;
 
 
 public class Server extends Thread {
+	private static final int NUMBER_OF_SPEEDTEST_RESULTS = 3;
+	
 	final private String HTTP_USER_AGENT = "Java/" + System.getProperty("java.version");
 	private String base_url;
 	private final OkHttpClient httpClient;
@@ -136,8 +142,8 @@ public class Server extends Thread {
 						String in = response.body().string();
 						
 						try {
-							HeartBeatInfos heartBeartInfos = new Persister().read(HeartBeatInfos.class, in);
+							HeartBeatInfos heartBeatInfos = new Persister().read(HeartBeatInfos.class, in);
-							ServerCode serverCode = ServerCode.fromInt(heartBeartInfos.getStatus());
+							ServerCode serverCode = ServerCode.fromInt(heartBeatInfos.getStatus());
 							if (serverCode == ServerCode.KEEPMEALIVE_STOP_RENDERING) {
 								this.log.debug("Server::stayAlive server asked to kill local render process");
 								// kill the current process, it will generate an error but it's okay
@@ -252,6 +258,41 @@ public class Server extends Thread {
 			return Error.Type.UNKNOWN;
 		}
 		
+		if (serverConfig.getSpeedTestTargets() != null && serverConfig.getSpeedTestTargets().isEmpty() == false) {
+			try {
+				client.getGui().status("Checking mirror connection speeds...");
+				Speedtest speedtest = new Speedtest(log);
+				List<SpeedTestTarget> bestSpeedTestTargets = speedtest.doSpeedtests(serverConfig.getSpeedTestTargets().stream().map(m -> m.getUrl()).collect(Collectors.toList()),
+					NUMBER_OF_SPEEDTEST_RESULTS);
+				SpeedTestResult result = new SpeedTestResult();
+				result.setResults(bestSpeedTestTargets.stream().map(m -> {
+					SpeedTestTargetResult targetResult = new SpeedTestTargetResult();
+					targetResult.setTarget(m.getUrl());
+					targetResult.setSpeed(m.getSpeedtest());
+					targetResult.setPing((int) (m.getPing().getAverage()));
+					return targetResult;
+				}).collect(Collectors.toList()));
+				
+				final Persister persister = new Persister();
+				try (StringWriter writer = new StringWriter()) {
+					persister.write(result, writer);
+					
+					HttpUrl.Builder urlBuilder = Objects.requireNonNull(HttpUrl.parse(this.getPage("speedtest-answer"))).newBuilder();
+					Response response = this.HTTPRequest(urlBuilder, RequestBody.create(MediaType.parse("application/xml"), writer.toString()));
+					if (response.code() != HttpURLConnection.HTTP_OK) {
+						throw new IOException("Server::getConfiguration Speedtest unexpected response");
+					}
+				}
+				catch (final Exception e) {
+					throw new IOException("Server::getConfiguration Speedtest failed to generate payload");
+				}
+			}
+			catch (IOException e) {
+				this.log.error("Server::getConfiguration Speedtest failed: " + e);
+				return Error.Type.NETWORK_ISSUE;
+			}
+		}
+		
 		client.setSessionStarted(true);
 		this.client.getGui().successfulAuthenticationEvent(publickey);
 		

src/com/sheepit/client/Speedtest.java

@@ -0,0 +1,129 @@
+package com.sheepit.client;
+
+import com.sheepit.client.datamodel.SpeedTestTarget;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.Socket;
+import java.net.URL;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
+public class Speedtest {
+	public static final int PORT = 443;
+	private static final Comparator<SpeedTestTarget> ORDERED = Comparator.comparing(speedTestTarget -> speedTestTarget.getPing().getAverage());
+	
+	private Log log;
+	
+	public Speedtest(Log log) {
+		this.log = log;
+	}
+
+	/**
+	* @param urls the urls to the speedtest payloads
+	* @param numberOfResults number of best mirrors to return
+	*
+	* @return An array of the mirrors with the best connection time. The size of the array is determined by <code>numberOfResults</code> or <code>urls.size()</code>
+	 * if <code>numberOfResults > urls.size()</code>
+	*/
+	public List<SpeedTestTarget> doSpeedtests(List<String> urls, int numberOfResults) {
+
+		List<SpeedTestTarget> pingResult = (urls
+				.stream()
+				.map(this::measure)
+				.sorted(ORDERED)
+				.collect(Collectors.toList())
+		);
+
+		numberOfResults = Math.min(numberOfResults, urls.size());
+		
+		List<SpeedTestTarget> result = new ArrayList<>(numberOfResults);
+		
+		int i = 0;
+		while (result.size() < numberOfResults && i < pingResult.size()) {
+			SpeedTestTarget m = pingResult.get(i);
+			try {
+				var speedtestResult = runTimed(() -> speedtest(m.getUrl()));
+				m.setSpeedtest(
+					Math.round(speedtestResult.second / (speedtestResult.first / (double) 1000L))	// number of bytes / time in seconds
+				);
+			}
+			catch (Exception e) {
+				this.log.error("Speedtest::doSpeedtests Exception " + e);
+				i++;
+				continue;
+			}
+			result.add(m);
+			i++;
+		}
+		
+		result.sort(Comparator.comparing(SpeedTestTarget::getSpeedtest).reversed());
+		return result;
+	}
+
+	private SpeedTestTarget measure(String mirror) {
+		long pingCount = 12;
+		var pingStatistics = LongStream
+				.range(0, pingCount)
+				.map(i -> {
+					try {
+						return runTimed(() -> ping(mirror, PORT)).first;
+					}
+					catch (Exception e) {
+						this.log.error("Speedtest::ping Exception " + e);
+						return Long.MAX_VALUE;
+					}
+				})
+				.summaryStatistics();
+
+		return new SpeedTestTarget(mirror, -1, pingStatistics);
+	}
+	
+	/**
+	 * Will return both the time it took to complete the task and the result of it
+	 * @param task the task whose execution time we want to measure
+	 * @param <T> the return value of the task
+	 * @return A pair where the first value is the execution time in ms and the second value is the task result
+	 * @throws Exception
+	 */
+	private <T> Pair<Long, T> runTimed(Callable<T> task) throws Exception {
+		long start, end;
+		start = System.nanoTime();
+		T callValue = task.call();
+		end = System.nanoTime();
+		return new Pair<>(Duration.ofNanos(end - start).toMillis(), callValue);
+	}
+	
+	/**
+	 * Downloads a payload from the given url and returns the number of downloaded bytes
+	 * @param url the url pointing at the speedtest file
+	 * @return the number of bytes read
+	 */
+	private int speedtest(String url) {
+		try (InputStream stream = new URL(url).openStream()) {
+			return stream.readAllBytes().length;
+		}
+		catch (MalformedURLException e) {
+			throw new RuntimeException("Invalid speedtest URL: " + url, e);
+		}
+		catch (IOException e) {
+			throw new RuntimeException("Unable to execute speedtest to: " + url, e);
+		}
+	}
+
+	private static int ping(String url, int port) {
+		try (Socket socket = new Socket(new URL(url).getHost(), port)) {
+		
+		}
+		catch (IOException e) {
+			throw new RuntimeException("Unable to open a socket to " + url + ":" + port, e);
+		}
+		return -1;
+	}
+}

src/com/sheepit/client/datamodel/ServerConfig.java

@@ -1,6 +1,7 @@
 package com.sheepit.client.datamodel;
 
 import lombok.Getter;
+import lombok.Setter;
 import lombok.ToString;
 import org.simpleframework.xml.Attribute;
 import org.simpleframework.xml.ElementList;
@@ -16,6 +17,9 @@ import java.util.List;
 	
 	@ElementList(name = "request", inline = true, required = false) private List<RequestEndPoint> requestEndPoints;
 	
+	@Getter @Setter
+	@ElementList(name = "speedtest", required = false) private List<SpeedTestTarget> speedTestTargets;
+	
 	public ServerConfig() {
 	}
 	

src/com/sheepit/client/datamodel/SpeedTestResult.java

@@ -0,0 +1,18 @@
+package com.sheepit.client.datamodel;
+
+
+import lombok.Data;
+import lombok.ToString;
+import org.simpleframework.xml.ElementList;
+import org.simpleframework.xml.Root;
+
+import java.util.List;
+
+@Root(strict = false, name = "speedtest") @Data @ToString public class SpeedTestResult {
+	
+	@ElementList(inline = true) private List<SpeedTestTargetResult> results;
+	
+	public SpeedTestResult() {
+	}
+}
+

src/com/sheepit/client/datamodel/SpeedTestTarget.java

@@ -0,0 +1,22 @@
+package com.sheepit.client.datamodel;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.simpleframework.xml.Attribute;
+import org.simpleframework.xml.Root;
+
+import java.util.LongSummaryStatistics;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Root(name = "target")
+public class SpeedTestTarget {
+
+    @Attribute(name = "url")
+    private String url;
+    private long speedtest;
+    private LongSummaryStatistics ping;
+
+}

src/com/sheepit/client/datamodel/SpeedTestTargetResult.java

@@ -0,0 +1,18 @@
+package com.sheepit.client.datamodel;
+
+import lombok.Data;
+import lombok.ToString;
+import org.simpleframework.xml.Attribute;
+import org.simpleframework.xml.Root;
+
+@Root(strict = false, name = "result") @Data @ToString public class SpeedTestTargetResult {
+	
+	@Attribute private String target;
+	
+	@Attribute private Long speed;
+	
+	@Attribute private Integer ping;
+	
+	public SpeedTestTargetResult() {
+	}
+}