From 2e466850689ca82653c975f6487eb89faf213d29 Mon Sep 17 00:00:00 2001 From: harlekin <5800926-mw102@users.noreply.gitlab.com> Date: Mon, 15 Nov 2021 14:09:41 +0000 Subject: [PATCH] feature: mirror speedtest --- protocol.txt | 123 ++++++++++++----- src/com/sheepit/client/Server.java | 45 +++++- src/com/sheepit/client/Speedtest.java | 129 ++++++++++++++++++ .../client/datamodel/ServerConfig.java | 4 + .../client/datamodel/SpeedTestResult.java | 18 +++ .../client/datamodel/SpeedTestTarget.java | 22 +++ .../datamodel/SpeedTestTargetResult.java | 18 +++ 7 files changed, 324 insertions(+), 35 deletions(-) create mode 100644 src/com/sheepit/client/Speedtest.java create mode 100644 src/com/sheepit/client/datamodel/SpeedTestResult.java create mode 100644 src/com/sheepit/client/datamodel/SpeedTestTarget.java create mode 100644 src/com/sheepit/client/datamodel/SpeedTestTargetResult.java diff --git a/protocol.txt b/protocol.txt index 8ace0af..ec70b0e 100644 --- a/protocol.txt +++ b/protocol.txt @@ -31,21 +31,31 @@ where X: * something else => Unknown error. Answer with no error: -A status of "0" to specify everything is okay, plus a list of URL paths for request, validation job, etc. -A path is provided for error, job request, job validation, download needed file, heartbeat (keepmealive), logout, thumbnail of last frame rendered and how many credits have been earned so far. +A status of "0" to specify everything is okay, publickey contains the public key for login, generated by the server, plus a list of URL paths for request, validation job, etc. +A path is provided for error, job request, job validation, download needed file, heartbeat (keepmealive), logout and speedtest results. +Also a speedtest element containing a url for each mirror leading to a payload for the ensuing speedtest. The maximum duration between two heartbeats in seconds is given by the attribute "max-period". + - + - + - - - + + + + + + + + + + + + - === Session end === Url: use the request type "logout" from the configuration answer. @@ -79,8 +89,12 @@ On error: an 404 http code Url: use the request type "request-job" from the configuration answer. Parameter as GET or POST: - * computemethod: What compute types are available on this machine. 0 for CPU or GPU, 1 for CPU only, 2 for GPU only. + * computemethod: What compute types are available on this machine. 0 for CPU or GPU, 1 for CPU only, 2 for GPU only. + * network_dl: Download speed in bytes/second + * network_up: Upload speed in bytes/second * cpu_cores: Number of cores currently available for rendering (optional). + * ram_max: Maximum memory allowed for renderer (in kilobytes). + * rendertime_max: Maximum allowed render time in seconds, 0 means no time limit * gpu_type: GPU's type, usually CUDA or OPENCL * gpu_model: Model name of the GPU available for rendering * gpu_ram: GPU memory size (in bytes) @@ -102,25 +116,43 @@ where X: Answer with no error: - - - - - + + + + + +stats => Some statistics about the session and the farm + * where + * credits_session => The points earned in this session + * credits_total => Represents the total amount of points of this account + * frame_remaining => How many frames are left to be rendered on the farm in total + * waiting_project => How many projects are in the render queue + * connected_machines => How many machines are connected to the farm right now + * renderable_projects => How many projects are available to be rendered by this session/machine +job => an element containing information about the new job + * where + * job id => The job id + * use_gpu => A flag selecting the render device (CPU/GPU) according to the session settings and the project (for example when a CPU+GPU session receives a CPU only project), 0 means false, 1 true + * archive_md5 => The MD5 checksum of the project zip, used to verify integrity after download + * path => The path to the blend file within the archive + * frame => The frame number to be rendered + * synchronous_upload => A flag indicating whether the job result can be queued up for uploading (0) or should be sent back immediately (1, only for the test frames) + * extras => Additional information, like whether the frame is a tile + * validation_url => The URL to send the job validation to + * name => The project name that will be displayed in the ui + * password => The archive password + renderer => An element containing information about the blender binary used for the job + * where + * md5 => The MD5 checksum of the binary archive, used to validate file integrity after download + * commandline => Launch arguments for the blender executable + * update_method => Whether to keep track of progress by remaining time (remainingtime) or render tile (by_tile) + + script => A server generated python script setting blender up with the right farm-related settings for the project === Job validation === @@ -137,7 +169,8 @@ Parameter as form-urlencoded: * file: the frame to send Answer in case of error: - + + where X: * 0 => No error * 300 => Missing parameter in request. @@ -168,18 +201,42 @@ where X: Url: use the request type "error" from the configuration answer. Parameter as GET or POST: + * type: X + where X: + * 1 => The client received a wrong configuration + * 2 => Authentication with www failed + * 3 => Client version too old + * 4 => The session got disabled + * 5 => Blender binary not available + * 6 => Blender binary missing + * 7 => Couldn't find a scene in the blend file + * 8 => Rendering produced no output file + * 9 => File download failed + * 10 => Can not create directory + * 11 => Network issue + * 12 => Renderer crashed + * 13 => Renderer failed due to running out of VRAM + * 14 => Render process got killed by OS + * 15 => Renderer is missing libraries + * 16 => Process execution failed + * 17 => OS not supported + * 18 => CPU not supported + * 19 => GPU not supported + * 20 => Renderer got killed by user + * 21 => Renderer failed due to running out of RAM + * 22 => Renderer got killed by server + * 23 => Renderer got killed due to exceeding the user time limit + * 24 => Renderer crashed with a python error + * 25 => Job validation failed + * 26 => Final image is too large + * 27 => Render engine not available + * 99 => Unknown error * job: job ID * frame: job's frame number * extras: job's extra data - * rendertime (optional): job's frame number + * render_time (optional): job's frame number * memoryused (optional): max memory used for the render (in kilo bytes) Parameter as form-urlencoded: * file: the error log to send -Answer: - - -where X: - * 0 => No error - diff --git a/src/com/sheepit/client/Server.java b/src/com/sheepit/client/Server.java index 11c2d14..bff84e1 100644 --- a/src/com/sheepit/client/Server.java +++ b/src/com/sheepit/client/Server.java @@ -35,7 +35,11 @@ import java.util.Date; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import com.sheepit.client.datamodel.SpeedTestTarget; +import com.sheepit.client.datamodel.SpeedTestResult; +import com.sheepit.client.datamodel.SpeedTestTargetResult; import lombok.Getter; import org.simpleframework.xml.core.Persister; @@ -74,6 +78,8 @@ import com.sheepit.client.os.OS; public class Server extends Thread { + private static final int NUMBER_OF_SPEEDTEST_RESULTS = 3; + final private String HTTP_USER_AGENT = "Java/" + System.getProperty("java.version"); private String base_url; private final OkHttpClient httpClient; @@ -136,8 +142,8 @@ public class Server extends Thread { String in = response.body().string(); try { - HeartBeatInfos heartBeartInfos = new Persister().read(HeartBeatInfos.class, in); - ServerCode serverCode = ServerCode.fromInt(heartBeartInfos.getStatus()); + HeartBeatInfos heartBeatInfos = new Persister().read(HeartBeatInfos.class, in); + ServerCode serverCode = ServerCode.fromInt(heartBeatInfos.getStatus()); if (serverCode == ServerCode.KEEPMEALIVE_STOP_RENDERING) { this.log.debug("Server::stayAlive server asked to kill local render process"); // kill the current process, it will generate an error but it's okay @@ -252,6 +258,41 @@ public class Server extends Thread { return Error.Type.UNKNOWN; } + if (serverConfig.getSpeedTestTargets() != null && serverConfig.getSpeedTestTargets().isEmpty() == false) { + try { + client.getGui().status("Checking mirror connection speeds..."); + Speedtest speedtest = new Speedtest(log); + List bestSpeedTestTargets = speedtest.doSpeedtests(serverConfig.getSpeedTestTargets().stream().map(m -> m.getUrl()).collect(Collectors.toList()), + NUMBER_OF_SPEEDTEST_RESULTS); + SpeedTestResult result = new SpeedTestResult(); + result.setResults(bestSpeedTestTargets.stream().map(m -> { + SpeedTestTargetResult targetResult = new SpeedTestTargetResult(); + targetResult.setTarget(m.getUrl()); + targetResult.setSpeed(m.getSpeedtest()); + targetResult.setPing((int) (m.getPing().getAverage())); + return targetResult; + }).collect(Collectors.toList())); + + final Persister persister = new Persister(); + try (StringWriter writer = new StringWriter()) { + persister.write(result, writer); + + HttpUrl.Builder urlBuilder = Objects.requireNonNull(HttpUrl.parse(this.getPage("speedtest-answer"))).newBuilder(); + Response response = this.HTTPRequest(urlBuilder, RequestBody.create(MediaType.parse("application/xml"), writer.toString())); + if (response.code() != HttpURLConnection.HTTP_OK) { + throw new IOException("Server::getConfiguration Speedtest unexpected response"); + } + } + catch (final Exception e) { + throw new IOException("Server::getConfiguration Speedtest failed to generate payload"); + } + } + catch (IOException e) { + this.log.error("Server::getConfiguration Speedtest failed: " + e); + return Error.Type.NETWORK_ISSUE; + } + } + client.setSessionStarted(true); this.client.getGui().successfulAuthenticationEvent(publickey); diff --git a/src/com/sheepit/client/Speedtest.java b/src/com/sheepit/client/Speedtest.java new file mode 100644 index 0000000..a009df4 --- /dev/null +++ b/src/com/sheepit/client/Speedtest.java @@ -0,0 +1,129 @@ +package com.sheepit.client; + +import com.sheepit.client.datamodel.SpeedTestTarget; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.Socket; +import java.net.URL; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +public class Speedtest { + public static final int PORT = 443; + private static final Comparator ORDERED = Comparator.comparing(speedTestTarget -> speedTestTarget.getPing().getAverage()); + + private Log log; + + public Speedtest(Log log) { + this.log = log; + } + + /** + * @param urls the urls to the speedtest payloads + * @param numberOfResults number of best mirrors to return + * + * @return An array of the mirrors with the best connection time. The size of the array is determined by numberOfResults or urls.size() + * if numberOfResults > urls.size() + */ + public List doSpeedtests(List urls, int numberOfResults) { + + List pingResult = (urls + .stream() + .map(this::measure) + .sorted(ORDERED) + .collect(Collectors.toList()) + ); + + numberOfResults = Math.min(numberOfResults, urls.size()); + + List result = new ArrayList<>(numberOfResults); + + int i = 0; + while (result.size() < numberOfResults && i < pingResult.size()) { + SpeedTestTarget m = pingResult.get(i); + try { + var speedtestResult = runTimed(() -> speedtest(m.getUrl())); + m.setSpeedtest( + Math.round(speedtestResult.second / (speedtestResult.first / (double) 1000L)) // number of bytes / time in seconds + ); + } + catch (Exception e) { + this.log.error("Speedtest::doSpeedtests Exception " + e); + i++; + continue; + } + result.add(m); + i++; + } + + result.sort(Comparator.comparing(SpeedTestTarget::getSpeedtest).reversed()); + return result; + } + + private SpeedTestTarget measure(String mirror) { + long pingCount = 12; + var pingStatistics = LongStream + .range(0, pingCount) + .map(i -> { + try { + return runTimed(() -> ping(mirror, PORT)).first; + } + catch (Exception e) { + this.log.error("Speedtest::ping Exception " + e); + return Long.MAX_VALUE; + } + }) + .summaryStatistics(); + + return new SpeedTestTarget(mirror, -1, pingStatistics); + } + + /** + * Will return both the time it took to complete the task and the result of it + * @param task the task whose execution time we want to measure + * @param the return value of the task + * @return A pair where the first value is the execution time in ms and the second value is the task result + * @throws Exception + */ + private Pair runTimed(Callable task) throws Exception { + long start, end; + start = System.nanoTime(); + T callValue = task.call(); + end = System.nanoTime(); + return new Pair<>(Duration.ofNanos(end - start).toMillis(), callValue); + } + + /** + * Downloads a payload from the given url and returns the number of downloaded bytes + * @param url the url pointing at the speedtest file + * @return the number of bytes read + */ + private int speedtest(String url) { + try (InputStream stream = new URL(url).openStream()) { + return stream.readAllBytes().length; + } + catch (MalformedURLException e) { + throw new RuntimeException("Invalid speedtest URL: " + url, e); + } + catch (IOException e) { + throw new RuntimeException("Unable to execute speedtest to: " + url, e); + } + } + + private static int ping(String url, int port) { + try (Socket socket = new Socket(new URL(url).getHost(), port)) { + + } + catch (IOException e) { + throw new RuntimeException("Unable to open a socket to " + url + ":" + port, e); + } + return -1; + } +} diff --git a/src/com/sheepit/client/datamodel/ServerConfig.java b/src/com/sheepit/client/datamodel/ServerConfig.java index 71b84be..1b63734 100644 --- a/src/com/sheepit/client/datamodel/ServerConfig.java +++ b/src/com/sheepit/client/datamodel/ServerConfig.java @@ -1,6 +1,7 @@ package com.sheepit.client.datamodel; import lombok.Getter; +import lombok.Setter; import lombok.ToString; import org.simpleframework.xml.Attribute; import org.simpleframework.xml.ElementList; @@ -16,6 +17,9 @@ import java.util.List; @ElementList(name = "request", inline = true, required = false) private List requestEndPoints; + @Getter @Setter + @ElementList(name = "speedtest", required = false) private List speedTestTargets; + public ServerConfig() { } diff --git a/src/com/sheepit/client/datamodel/SpeedTestResult.java b/src/com/sheepit/client/datamodel/SpeedTestResult.java new file mode 100644 index 0000000..5c28ba3 --- /dev/null +++ b/src/com/sheepit/client/datamodel/SpeedTestResult.java @@ -0,0 +1,18 @@ +package com.sheepit.client.datamodel; + + +import lombok.Data; +import lombok.ToString; +import org.simpleframework.xml.ElementList; +import org.simpleframework.xml.Root; + +import java.util.List; + +@Root(strict = false, name = "speedtest") @Data @ToString public class SpeedTestResult { + + @ElementList(inline = true) private List results; + + public SpeedTestResult() { + } +} + diff --git a/src/com/sheepit/client/datamodel/SpeedTestTarget.java b/src/com/sheepit/client/datamodel/SpeedTestTarget.java new file mode 100644 index 0000000..e9f7010 --- /dev/null +++ b/src/com/sheepit/client/datamodel/SpeedTestTarget.java @@ -0,0 +1,22 @@ +package com.sheepit.client.datamodel; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.simpleframework.xml.Attribute; +import org.simpleframework.xml.Root; + +import java.util.LongSummaryStatistics; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Root(name = "target") +public class SpeedTestTarget { + + @Attribute(name = "url") + private String url; + private long speedtest; + private LongSummaryStatistics ping; + +} diff --git a/src/com/sheepit/client/datamodel/SpeedTestTargetResult.java b/src/com/sheepit/client/datamodel/SpeedTestTargetResult.java new file mode 100644 index 0000000..09fb3cf --- /dev/null +++ b/src/com/sheepit/client/datamodel/SpeedTestTargetResult.java @@ -0,0 +1,18 @@ +package com.sheepit.client.datamodel; + +import lombok.Data; +import lombok.ToString; +import org.simpleframework.xml.Attribute; +import org.simpleframework.xml.Root; + +@Root(strict = false, name = "result") @Data @ToString public class SpeedTestTargetResult { + + @Attribute private String target; + + @Attribute private Long speed; + + @Attribute private Integer ping; + + public SpeedTestTargetResult() { + } +}