Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ref: move directories handle to a specific class (outside of Client class)

Browse Source
This commit is contained in:
Sheepit Renderfarm
2023-12-19 14:35:24 +00:00
parent b4d7ee5edb
commit 3e850ad6e5
3 changed files with 123 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/com/sheepit/client/Client.java
View File

@@ -24,11 +24,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@@ -66,6 +61,7 @@ import okhttp3.HttpUrl;
@Data public class Client {
public static final int MIN_JOB_ID = 20; //to distinguish between actual jobs and test frames
private static final Locale LOCALE = Locale.ENGLISH;
private DirectoryManager directoryManager;
private Gui gui;
private Server server;
private Configuration configuration;
@@ -92,6 +88,7 @@ import okhttp3.HttpUrl;
this.server = new Server(url_, this.configuration, this);
this.log = Log.getInstance(this.configuration);
this.gui = gui_;
this.directoryManager = new DirectoryManager(this.configuration, this.log);
this.renderingJob = null;
this.previousJob = null;
this.jobsToValidate = new ArrayBlockingQueue<QueuedJob>(5);
@@ -793,7 +790,7 @@ import okhttp3.HttpUrl;
this.gui,
this.log,
String.format(LOCALE, "chunk %d/%d", i + 1, total),
ajob_.getRequiredProjectChunkPath(chunk.getId()),
this.directoryManager.getActualStoragePathFor(chunk),
chunk.getMd5(),
String.format(LOCALE, "%s?chunk=%s", this.server.getPage("download-chunk"), chunk.getId())
);
@@ -811,7 +808,7 @@ import okhttp3.HttpUrl;
this.gui,
this.log,
"renderer",
ajob.getRequiredRendererArchivePath(),
this.directoryManager.getActualStorageBinaryPathFor(ajob),
ajob.getRendererMD5(),
String.format(LOCALE, "%s?job=%s", this.server.getPage("download-binary"), ajob.getId())
)).download();
@@ -823,15 +820,18 @@ import okhttp3.HttpUrl;
protected int prepareWorkingDirectory(Job ajob) {
int ret;
String bestRendererArchive = ajob.getRequiredRendererArchivePath();
String renderer_archive = ajob.getRendererArchivePath();
String renderer_archive = this.directoryManager.getCacheBinaryPathFor(ajob);
String renderer_path = ajob.getRendererDirectory();
File renderer_path_file = new File(renderer_path);
if (!new File(renderer_archive).exists()) {
// file is already downloaded, either on shared directory or cache directory (from this.downloadExecutable)
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedBinaryPathFor(ajob)).exists()) {
this.gui.status("Copying renderer from shared downloads directory");
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + bestRendererArchive + " into " + renderer_archive);
copySharedChunk(bestRendererArchive, renderer_archive);
this.log.debug("Client::prepareWorkingDirectory Copying renderer from shared downloads directory " + this.directoryManager.getSharedBinaryPathFor(ajob) + " into " + this.directoryManager.getCacheBinaryPathFor(ajob));
if (this.directoryManager.copyBinaryFromSharedToCache(ajob) == false) {
log.error("Error while copying " + renderer_archive + " from shared downloads directory to working dir");
}
}
if (!renderer_path_file.exists()) {
@@ -862,11 +862,15 @@ import okhttp3.HttpUrl;
String scene_path = ajob.getSceneDirectory();
File scene_path_file = new File(scene_path);
// chunk files are already downloaded, either on shared directory or cache directory (from this.downloadSceneFile)
for (Chunk chunk: ajob.getArchiveChunks()) {
if (new File(ajob.getRequiredProjectChunkPath(chunk.getId())).exists()) {
if (this.directoryManager.isSharedEnabled() && new File(this.directoryManager.getSharedPathFor(chunk)).exists()) {
this.gui.status("Copying chunk from common directory");
copySharedChunk(ajob.getRequiredProjectChunkPath(chunk.getId()), ajob.getSceneArchiveChunkPath(chunk.getId()));
if (this.directoryManager.copyChunkFromSharedToCache(chunk) == false) {
this.log.error("Error while copying " + this.directoryManager.getSharedPathFor(chunk) + " from shared downloads directory to working dir");
}
}
}
/// download the chunks
@@ -881,7 +885,7 @@ import okhttp3.HttpUrl;
ret = Utils.unzipChunksIntoDirectory(
ajob.getArchiveChunks().stream().map(input -> ajob.getSceneArchiveChunkPath(input.getId())).collect(Collectors.toList()),
ajob.getArchiveChunks().stream().map(chunk -> this.directoryManager.getCachePathFor(chunk)).collect(Collectors.toList()),
scene_path,
ajob.getPassword(),
log);
@@ -895,28 +899,6 @@ import okhttp3.HttpUrl;
return 0;
}
private void copySharedChunk(String existingArchive, String targetArchive) {
Path existingArchivePath = Paths.get(existingArchive);
Path targetArchivePath = Paths.get(targetArchive);
try {
try {
Files.createLink(targetArchivePath, existingArchivePath);
log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath);
}
catch (UnsupportedOperationException // underlying file system does not support hard-linking
| FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported
| SecurityException // user is not allowed to create hard-links
ignore) {
// Creating hardlinks might not be supported on some filesystems
log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
}
}
catch (IOException e) {
this.gui.error("Error while copying " + existingArchive + " from shared downloads directory to working dir");
}
}
protected Error.Type confirmJob(Job ajob, int checkpoint) {
String url_real = String.format(LOCALE, "%s&rendertime=%d&memoryused=%s", ajob.getValidationUrl(), ajob.getProcessRender().getRenderDuration(),
ajob.getProcessRender().getPeakMemoryUsed());

104
src/main/java/com/sheepit/client/DirectoryManager.java Normal file
View File

@@ -0,0 +1,104 @@
/*
* Copyright (C) 2023 Laurent CLOUET
* Author Laurent CLOUET <laurent.clouet@nopnop.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; version 2
* of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package com.sheepit.client;
import com.sheepit.client.datamodel.Chunk;
import lombok.AllArgsConstructor;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
@AllArgsConstructor
public class DirectoryManager {
private Configuration configuration;
private Log log;
public String getActualStoragePathFor(Chunk chunk) {
return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk);
}
public String getActualStorageBinaryPathFor(Job job) {
return isSharedEnabled() ? getSharedBinaryPathFor(job) : getCacheBinaryPathFor(job);
}
public String getCachePathFor(Chunk chunk) {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool";
}
public String getSharedPathFor(Chunk chunk) {
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk.getId() + ".wool";
}
public String getCacheBinaryPathFor(Job job) {
return configuration.getStorageDir().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip";
}
public String getSharedBinaryPathFor(Job job) {
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + job.getRendererMD5() + ".zip";
}
public boolean isSharedEnabled() {
return configuration.getSharedDownloadsDirectory() != null && configuration.getSharedDownloadsDirectory().exists();
}
public boolean copyBinaryFromSharedToCache(Job job) {
return copyFileFromSharedToCache(getSharedBinaryPathFor(job), getCacheBinaryPathFor(job));
}
public boolean copyChunkFromSharedToCache(Chunk chunk) {
return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk));
}
private boolean copyFileFromSharedToCache(String source, String target) {
Path existingArchivePath = Paths.get(source);
Path targetArchivePath = Paths.get(target);
if (existingArchivePath.equals(targetArchivePath)) {
// target are the same, do nothing
return true;
}
try {
try {
Files.createLink(targetArchivePath, existingArchivePath);
log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath);
}
catch (UnsupportedOperationException // underlying file system does not support hard-linking
| FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported
| SecurityException // user is not allowed to create hard-links
ignore) {
// Creating hardlinks might not be supported on some filesystems
log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
}
}
catch (IOException e) {
log.error("Error while copying " + source + " from shared downloads directory to working dir");
return false;
}
return true;
}
}

26
src/main/java/com/sheepit/client/Job.java
View File

@@ -147,32 +147,10 @@ import java.util.regex.Pattern;
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + rendererMD5;
}
public String getRequiredRendererArchivePath() {
if (configuration.getSharedDownloadsDirectory() != null) {
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
}
else {
return getRendererArchivePath();
}
}
public String getRendererPath() {
return getRendererDirectory() + File.separator + OS.getOS().getRenderBinaryPath();
}
public String getRendererArchivePath() {
return configuration.getStorageDir().getAbsolutePath() + File.separator + rendererMD5 + ".zip";
}
public String getRequiredProjectChunkPath(String chunk) {
if (configuration.getSharedDownloadsDirectory() != null) {
return configuration.getSharedDownloadsDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
}
else {
return getSceneArchiveChunkPath(chunk);
}
}
public String getSceneDirectory() {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + this.id;
}
@@ -181,10 +159,6 @@ import java.util.regex.Pattern;
return getSceneDirectory() + File.separator + this.path;
}
public String getSceneArchiveChunkPath(String chunk) {
return configuration.getWorkingDirectory().getAbsolutePath() + File.separator + chunk + ".wool";
}
public Error.Type render(Observer renderStarted) {
gui.status("Rendering");
RenderProcess process = getProcessRender();