Ref: configuration class shouldn't create and handle directory, DirectoryManager should do it
This commit is contained in:
@@ -96,7 +96,7 @@ import okhttp3.HttpUrl;
|
|||||||
this.server = new Server(url, this.configuration, this);
|
this.server = new Server(url, this.configuration, this);
|
||||||
this.log = Log.getInstance(this.configuration);
|
this.log = Log.getInstance(this.configuration);
|
||||||
this.gui = gui;
|
this.gui = gui;
|
||||||
this.directoryManager = new DirectoryManager(this.configuration, this.log);
|
this.directoryManager = new DirectoryManager(this.configuration);
|
||||||
this.renderingJob = null;
|
this.renderingJob = null;
|
||||||
this.previousJob = null;
|
this.previousJob = null;
|
||||||
this.jobsToValidate = new ArrayBlockingQueue<>(5);
|
this.jobsToValidate = new ArrayBlockingQueue<>(5);
|
||||||
@@ -171,13 +171,13 @@ import okhttp3.HttpUrl;
|
|||||||
step = log.newCheckPoint();
|
step = log.newCheckPoint();
|
||||||
this.log.info("OS: " + OS.getOS().getVersion() + " " + System.getProperty("os.arch"));
|
this.log.info("OS: " + OS.getOS().getVersion() + " " + System.getProperty("os.arch"));
|
||||||
this.log.info(configuration.toString());
|
this.log.info(configuration.toString());
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
this.log.info(logline);
|
this.log.info(logline);
|
||||||
}
|
}
|
||||||
sendError(step, null, Type.OK);
|
sendError(step, null, Type.OK);
|
||||||
|
|
||||||
// Check integrity of all files in the working directories
|
// Check integrity of all files in the working directories
|
||||||
this.configuration.cleanWorkingDirectory();
|
this.directoryManager.cleanWorkingDirectory();
|
||||||
|
|
||||||
this.startTime = new Date().getTime();
|
this.startTime = new Date().getTime();
|
||||||
this.server.start(); // for staying alive
|
this.server.start(); // for staying alive
|
||||||
@@ -435,7 +435,7 @@ import okhttp3.HttpUrl;
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.configuration.removeWorkingDirectory();
|
this.directoryManager.removeWorkingDirectory();
|
||||||
|
|
||||||
if (this.server == null) {
|
if (this.server == null) {
|
||||||
return 0;
|
return 0;
|
||||||
@@ -703,7 +703,7 @@ import okhttp3.HttpUrl;
|
|||||||
downloadRet = this.downloadExecutable(ajob);
|
downloadRet = this.downloadExecutable(ajob);
|
||||||
if (downloadRet != Error.Type.OK) {
|
if (downloadRet != Error.Type.OK) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
this.log.error("Client::work problem with downloadExecutable (ret " + downloadRet + ")");
|
this.log.error("Client::work problem with downloadExecutable (ret " + downloadRet + ")");
|
||||||
@@ -713,7 +713,7 @@ import okhttp3.HttpUrl;
|
|||||||
downloadRet = this.downloadSceneFile(ajob);
|
downloadRet = this.downloadSceneFile(ajob);
|
||||||
if (downloadRet != Error.Type.OK) {
|
if (downloadRet != Error.Type.OK) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
this.log.error("Client::work problem with downloadSceneFile (ret " + downloadRet + ")");
|
this.log.error("Client::work problem with downloadSceneFile (ret " + downloadRet + ")");
|
||||||
@@ -723,7 +723,7 @@ import okhttp3.HttpUrl;
|
|||||||
int ret = this.prepareWorkingDirectory(ajob); // decompress renderer and scene archives
|
int ret = this.prepareWorkingDirectory(ajob); // decompress renderer and scene archives
|
||||||
if (ret != 0) {
|
if (ret != 0) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
this.log.error("Client::work problem with this.prepareWorkingDirectory (ret " + ret + ")");
|
this.log.error("Client::work problem with this.prepareWorkingDirectory (ret " + ret + ")");
|
||||||
@@ -732,7 +732,7 @@ import okhttp3.HttpUrl;
|
|||||||
}
|
}
|
||||||
catch (SheepItException e) {
|
catch (SheepItException e) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
if (e instanceof SheepItExceptionNoSpaceLeftOnDevice) {
|
if (e instanceof SheepItExceptionNoSpaceLeftOnDevice) {
|
||||||
@@ -754,23 +754,23 @@ import okhttp3.HttpUrl;
|
|||||||
|
|
||||||
if (sceneFile.exists() == false) {
|
if (sceneFile.exists() == false) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
this.log.error("Client::work job preparation failed (scene file '" + sceneFile.getAbsolutePath()
|
this.log.error("Client::work job preparation failed (scene file '" + sceneFile.getAbsolutePath()
|
||||||
+ "' does not exist), cleaning directory in hope to recover");
|
+ "' does not exist), cleaning directory in hope to recover");
|
||||||
this.configuration.cleanWorkingDirectory();
|
this.directoryManager.cleanWorkingDirectory();
|
||||||
return Error.Type.MISSING_SCENE;
|
return Error.Type.MISSING_SCENE;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rendererFile.exists() == false) {
|
if (rendererFile.exists() == false) {
|
||||||
gui.setRenderingProjectName("");
|
gui.setRenderingProjectName("");
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
this.log.error("Client::work job preparation failed (renderer file '" + rendererFile.getAbsolutePath()
|
this.log.error("Client::work job preparation failed (renderer file '" + rendererFile.getAbsolutePath()
|
||||||
+ "' does not exist), cleaning directory in hope to recover");
|
+ "' does not exist), cleaning directory in hope to recover");
|
||||||
this.configuration.cleanWorkingDirectory();
|
this.directoryManager.cleanWorkingDirectory();
|
||||||
return Error.Type.MISSING_RENDERER;
|
return Error.Type.MISSING_RENDERER;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,12 +26,8 @@ import java.io.InputStream;
|
|||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.UnknownHostException;
|
import java.net.UnknownHostException;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.ListIterator;
|
|
||||||
|
|
||||||
import com.sheepit.client.hardware.cpu.CPU;
|
import com.sheepit.client.hardware.cpu.CPU;
|
||||||
import com.sheepit.client.hardware.gpu.GPUDevice;
|
import com.sheepit.client.hardware.gpu.GPUDevice;
|
||||||
@@ -187,40 +183,26 @@ import lombok.Data;
|
|||||||
* @param cache_dir_ File representing cache directory
|
* @param cache_dir_ File representing cache directory
|
||||||
*/
|
*/
|
||||||
public void setCacheDir(File cache_dir_) {
|
public void setCacheDir(File cache_dir_) {
|
||||||
removeWorkingDirectory();
|
|
||||||
if (cache_dir_ == null) {
|
if (cache_dir_ == null) {
|
||||||
this.userHasSpecifiedACacheDir = false;
|
this.userHasSpecifiedACacheDir = false;
|
||||||
try {
|
try {
|
||||||
this.workingDirectory = File.createTempFile("farm_", "");
|
this.workingDirectory = File.createTempFile("farm_", "");
|
||||||
this.workingDirectory.createNewFile(); // hoho...
|
this.workingDirectory.createNewFile(); // hoho...
|
||||||
this.workingDirectory.delete(); // hoho
|
this.workingDirectory.delete(); // hoho
|
||||||
this.workingDirectory.mkdir();
|
|
||||||
this.workingDirectory.deleteOnExit();
|
|
||||||
|
|
||||||
// since there is no working directory and the client will be working in the system temp directory,
|
|
||||||
// we can also set up a 'permanent' directory for immutable files (like renderer binary)
|
|
||||||
|
|
||||||
this.woolCacheDirectory = new File(this.workingDirectory.getParent() + File.separator + WOOL_CACHE_DIRNAME);
|
|
||||||
this.woolCacheDirectory.mkdir();
|
|
||||||
}
|
}
|
||||||
catch (IOException e) {
|
catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// since there is no working directory and the client will be working in the system temp directory,
|
||||||
|
// we can also set up a 'permanent' directory for immutable files (like renderer binary)
|
||||||
|
this.woolCacheDirectory = new File(this.workingDirectory.getParent() + File.separator + WOOL_CACHE_DIRNAME);
|
||||||
|
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
this.userHasSpecifiedACacheDir = true;
|
this.userHasSpecifiedACacheDir = true;
|
||||||
this.workingDirectory = new File(cache_dir_.getAbsolutePath() + File.separator + WORKING_DIRNAME);
|
this.workingDirectory = new File(cache_dir_.getAbsolutePath() + File.separator + WORKING_DIRNAME);
|
||||||
this.woolCacheDirectory = new File(cache_dir_.getAbsolutePath() + File.separator + WOOL_CACHE_DIRNAME);
|
this.woolCacheDirectory = new File(cache_dir_.getAbsolutePath() + File.separator + WOOL_CACHE_DIRNAME);
|
||||||
this.workingDirectory.mkdirs();
|
|
||||||
this.woolCacheDirectory.mkdirs();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.sharedDownloadsDirectory != null) {
|
|
||||||
this.sharedDownloadsDirectory.mkdirs();
|
|
||||||
|
|
||||||
if (!this.sharedDownloadsDirectory.exists()) {
|
|
||||||
System.err.println("Configuration::setCacheDir Unable to create common directory " + this.sharedDownloadsDirectory.getAbsolutePath());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,175 +227,6 @@ import lombok.Data;
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Deletes the working and storage directories
|
|
||||||
*/
|
|
||||||
public void cleanWorkingDirectory() {
|
|
||||||
this.cleanDirectory(this.workingDirectory);
|
|
||||||
this.cleanDirectory(this.woolCacheDirectory);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleans a directory and removes files in it from the md5 cache
|
|
||||||
* @param dir representing the directory to be cleaned
|
|
||||||
* @return false if the dir null, true otherwise
|
|
||||||
*/
|
|
||||||
public boolean cleanDirectory(File dir) {
|
|
||||||
if (dir == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
File[] files = dir.listFiles();
|
|
||||||
if (files != null) {
|
|
||||||
for (File file : files) {
|
|
||||||
if (file.isDirectory()) {
|
|
||||||
Utils.delete(file);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
try {
|
|
||||||
String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
|
|
||||||
String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
|
|
||||||
if (".wool".equals(extension)) {
|
|
||||||
// check if the md5 of the file is ok
|
|
||||||
String md5_local = Utils.md5(file.getAbsolutePath());
|
|
||||||
|
|
||||||
if (md5_local.equals(name) == false) {
|
|
||||||
file.delete();
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: remove old one
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
file.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (StringIndexOutOfBoundsException e) { // because the file does not have an . in his path
|
|
||||||
file.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cleans working directory and also deletes it if the user hasn't specified a cache directory
|
|
||||||
*/
|
|
||||||
public void removeWorkingDirectory() {
|
|
||||||
if (this.userHasSpecifiedACacheDir) {
|
|
||||||
this.cleanWorkingDirectory();
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Utils.delete(this.workingDirectory);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return a list of archives (files with extensions .zip or .wool) in the
|
|
||||||
* working, storage, and shared downloads directories as long as they are not null
|
|
||||||
*/
|
|
||||||
public List<File> getLocalCacheFiles() {
|
|
||||||
List<File> files_local = new LinkedList<File>();
|
|
||||||
List<File> files = new LinkedList<File>();
|
|
||||||
if (this.workingDirectory != null) {
|
|
||||||
File[] filesInDirectory = this.workingDirectory.listFiles();
|
|
||||||
if (filesInDirectory != null) {
|
|
||||||
files.addAll(Arrays.asList(filesInDirectory));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (this.woolCacheDirectory != null) {
|
|
||||||
File[] filesInDirectory = this.woolCacheDirectory.listFiles();
|
|
||||||
if (filesInDirectory != null) {
|
|
||||||
files.addAll(Arrays.asList(filesInDirectory));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (this.sharedDownloadsDirectory != null) {
|
|
||||||
File[] filesInDirectory = this.sharedDownloadsDirectory.listFiles();
|
|
||||||
if (filesInDirectory != null) {
|
|
||||||
files.addAll(Arrays.asList(filesInDirectory));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (File file : files) {
|
|
||||||
if (file.isFile()) {
|
|
||||||
try {
|
|
||||||
String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
|
|
||||||
String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
|
|
||||||
if (".wool".equals(extension)) {
|
|
||||||
// check if the md5 of the file is ok
|
|
||||||
String md5_local = Utils.md5(file.getAbsolutePath());
|
|
||||||
|
|
||||||
if (md5_local.equals(name)) {
|
|
||||||
files_local.add(file);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (StringIndexOutOfBoundsException e) { // because the file does not have an . his path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return files_local;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Runs through all SheepIt related directories and checks if files and folders are all readable, writeable
|
|
||||||
* and in case of directories, checks if the contents can be listed and if usable space is enough.
|
|
||||||
* Only logs instances where something was detected, otherwise is it will only print "FilesystemHealthCheck started"
|
|
||||||
* @return an ArrayList of Strings containing all logs of the FSHealth check
|
|
||||||
*/
|
|
||||||
public List<String> filesystemHealthCheck() {
|
|
||||||
List<String> logs = new ArrayList<>();
|
|
||||||
String f = "FSHealth: ";
|
|
||||||
logs.add(f + "FilesystemHealthCheck started");
|
|
||||||
List<File> dirsToCheck = new ArrayList<>();
|
|
||||||
List<File> dirsChecked = new ArrayList<>();
|
|
||||||
dirsToCheck.add(workingDirectory.getAbsoluteFile());
|
|
||||||
if (sharedDownloadsDirectory != null && dirsToCheck.contains(sharedDownloadsDirectory.getAbsoluteFile()) == false) {
|
|
||||||
dirsToCheck.add(sharedDownloadsDirectory.getAbsoluteFile());
|
|
||||||
}
|
|
||||||
if (woolCacheDirectory != null && dirsToCheck.contains(woolCacheDirectory.getAbsoluteFile()) == false) {
|
|
||||||
dirsToCheck.add(woolCacheDirectory.getAbsoluteFile());
|
|
||||||
}
|
|
||||||
ListIterator<File> dirs = dirsToCheck.listIterator();
|
|
||||||
while (dirs.hasNext()) {
|
|
||||||
File dir = dirs.next();
|
|
||||||
dirs.remove();
|
|
||||||
dirsChecked.add(dir);
|
|
||||||
File[] fileList = dir.listFiles();
|
|
||||||
if (fileList == null) {
|
|
||||||
logs.add(f + "File list of " + dir + " is null");
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
for (File file : fileList) {
|
|
||||||
file = file.getAbsoluteFile();
|
|
||||||
//logs.add(f + file);
|
|
||||||
boolean canRead = file.canRead();
|
|
||||||
boolean canWrite = file.canWrite();
|
|
||||||
boolean isDir = file.isDirectory();
|
|
||||||
if (canRead == false) {
|
|
||||||
logs.add(f + "Can't read from " + file);
|
|
||||||
}
|
|
||||||
if (canWrite == false) {
|
|
||||||
logs.add(f + "Can't write to " + file);
|
|
||||||
}
|
|
||||||
if (canRead && canWrite && isDir) {
|
|
||||||
if (dirsChecked.contains(file)) {
|
|
||||||
logs.add(f + "Dir " + file + " already checked. Loop detected");
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
dirs.add(file);
|
|
||||||
}
|
|
||||||
long usableSpace = file.getUsableSpace();
|
|
||||||
if (usableSpace < 512 * 1024) {
|
|
||||||
logs.add(f + "Usable space is " + usableSpace + " for " + file);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return logs;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return a string representing the version of the jar or
|
* @return a string representing the version of the jar or
|
||||||
* {@code 6.0.0} if on error is encountered or the VERSION file doesn't exist
|
* {@code 6.0.0} if on error is encountered or the VERSION file doesn't exist
|
||||||
|
|||||||
@@ -29,11 +29,15 @@ import java.nio.file.Files;
|
|||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.nio.file.StandardCopyOption;
|
import java.nio.file.StandardCopyOption;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.ListIterator;
|
||||||
|
|
||||||
@AllArgsConstructor
|
@AllArgsConstructor
|
||||||
public class DirectoryManager {
|
public class DirectoryManager {
|
||||||
private Configuration configuration;
|
private Configuration configuration;
|
||||||
private Log log;
|
|
||||||
|
|
||||||
public String getActualStoragePathFor(Chunk chunk) {
|
public String getActualStoragePathFor(Chunk chunk) {
|
||||||
return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk);
|
return isSharedEnabled() ? getSharedPathFor(chunk) : getCachePathFor(chunk);
|
||||||
@@ -68,22 +72,209 @@ public class DirectoryManager {
|
|||||||
Files.deleteIfExists(targetArchivePath); // createLink only works if the target does not exist
|
Files.deleteIfExists(targetArchivePath); // createLink only works if the target does not exist
|
||||||
try {
|
try {
|
||||||
Files.createLink(targetArchivePath, existingArchivePath);
|
Files.createLink(targetArchivePath, existingArchivePath);
|
||||||
log.debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath);
|
Log.getInstance(configuration).debug("Created hardlink from " + targetArchivePath + " to " + existingArchivePath);
|
||||||
}
|
}
|
||||||
catch (UnsupportedOperationException // underlying file system does not support hard-linking
|
catch (UnsupportedOperationException // underlying file system does not support hard-linking
|
||||||
| FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported
|
| FileSystemException // cache-dir and shared-zip are on separate file systems, even though hard-linking is supported
|
||||||
| SecurityException // user is not allowed to create hard-links
|
| SecurityException // user is not allowed to create hard-links
|
||||||
ignore) {
|
ignore) {
|
||||||
// Creating hardlinks might not be supported on some filesystems
|
// Creating hardlinks might not be supported on some filesystems
|
||||||
log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
|
Log.getInstance(configuration).debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
|
||||||
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
|
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (IOException e) {
|
catch (IOException e) {
|
||||||
log.error("Error while copying " + source + " from shared downloads directory to working dir");
|
Log.getInstance(configuration).error("Error while copying " + source + " from shared downloads directory to working dir");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates cache directory
|
||||||
|
*/
|
||||||
|
public void createCacheDir() {
|
||||||
|
this.removeWorkingDirectory();
|
||||||
|
|
||||||
|
this.configuration.getWorkingDirectory().mkdirs();
|
||||||
|
this.configuration.getWoolCacheDirectory().mkdirs();
|
||||||
|
|
||||||
|
if (this.configuration.getSharedDownloadsDirectory() != null) {
|
||||||
|
this.configuration.getSharedDownloadsDirectory().mkdirs();
|
||||||
|
|
||||||
|
if (this.configuration.getSharedDownloadsDirectory().exists() == false) {
|
||||||
|
System.err.println("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleans working directory and also deletes it if the user hasn't specified a cache directory
|
||||||
|
*/
|
||||||
|
public void removeWorkingDirectory() {
|
||||||
|
if (this.configuration.isUserHasSpecifiedACacheDir()) {
|
||||||
|
this.cleanWorkingDirectory();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Utils.delete(this.configuration.getWorkingDirectory());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes the working and storage directories
|
||||||
|
*/
|
||||||
|
public void cleanWorkingDirectory() {
|
||||||
|
this.cleanDirectory(this.configuration.getWorkingDirectory());
|
||||||
|
this.cleanDirectory(this.configuration.getWoolCacheDirectory());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleans a directory and removes files in it from the md5 cache
|
||||||
|
* @param dir representing the directory to be cleaned
|
||||||
|
* @return false if the dir null, true otherwise
|
||||||
|
*/
|
||||||
|
private boolean cleanDirectory(File dir) {
|
||||||
|
if (dir == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
File[] files = dir.listFiles();
|
||||||
|
if (files != null) {
|
||||||
|
for (File file : files) {
|
||||||
|
if (file.isDirectory()) {
|
||||||
|
Utils.delete(file);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
try {
|
||||||
|
String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
|
||||||
|
String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
|
||||||
|
if (".wool".equals(extension)) {
|
||||||
|
// check if the md5 of the file is ok
|
||||||
|
String md5_local = Utils.md5(file.getAbsolutePath());
|
||||||
|
|
||||||
|
if (md5_local.equals(name) == false) {
|
||||||
|
file.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: remove old one
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
file.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (StringIndexOutOfBoundsException e) { // because the file does not have an . in his path
|
||||||
|
file.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return a list of archives (files with extensions .zip or .wool) in the
|
||||||
|
* working, storage, and shared downloads directories as long as they are not null
|
||||||
|
*/
|
||||||
|
public List<File> getLocalCacheFiles() {
|
||||||
|
List<File> files_local = new LinkedList<File>();
|
||||||
|
List<File> files = new LinkedList<File>();
|
||||||
|
if (this.configuration.getWorkingDirectory() != null) {
|
||||||
|
File[] filesInDirectory = this.configuration.getWorkingDirectory().listFiles();
|
||||||
|
if (filesInDirectory != null) {
|
||||||
|
files.addAll(Arrays.asList(filesInDirectory));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.configuration.getWoolCacheDirectory() != null) {
|
||||||
|
File[] filesInDirectory = this.configuration.getWoolCacheDirectory().listFiles();
|
||||||
|
if (filesInDirectory != null) {
|
||||||
|
files.addAll(Arrays.asList(filesInDirectory));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.configuration.getSharedDownloadsDirectory() != null) {
|
||||||
|
File[] filesInDirectory = this.configuration.getSharedDownloadsDirectory().listFiles();
|
||||||
|
if (filesInDirectory != null) {
|
||||||
|
files.addAll(Arrays.asList(filesInDirectory));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (File file : files) {
|
||||||
|
if (file.isFile()) {
|
||||||
|
try {
|
||||||
|
String extension = file.getName().substring(file.getName().lastIndexOf('.')).toLowerCase();
|
||||||
|
String name = file.getName().substring(0, file.getName().length() - 1 * extension.length());
|
||||||
|
if (".wool".equals(extension)) {
|
||||||
|
// check if the md5 of the file is ok
|
||||||
|
String md5_local = Utils.md5(file.getAbsolutePath());
|
||||||
|
|
||||||
|
if (md5_local.equals(name)) {
|
||||||
|
files_local.add(file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (StringIndexOutOfBoundsException e) { // because the file does not have an . his path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files_local;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs through all SheepIt related directories and checks if files and folders are all readable, writeable
|
||||||
|
* and in case of directories, checks if the contents can be listed and if usable space is enough.
|
||||||
|
* Only logs instances where something was detected, otherwise is it will only print "FilesystemHealthCheck started"
|
||||||
|
* @return an ArrayList of Strings containing all logs of the FSHealth check
|
||||||
|
*/
|
||||||
|
public List<String> filesystemHealthCheck() {
|
||||||
|
List<String> logs = new ArrayList<>();
|
||||||
|
String f = "FSHealth: ";
|
||||||
|
logs.add(f + "FilesystemHealthCheck started");
|
||||||
|
List<File> dirsToCheck = new ArrayList<>();
|
||||||
|
List<File> dirsChecked = new ArrayList<>();
|
||||||
|
dirsToCheck.add(configuration.getWorkingDirectory().getAbsoluteFile());
|
||||||
|
if (configuration.getSharedDownloadsDirectory() != null && dirsToCheck.contains(configuration.getSharedDownloadsDirectory().getAbsoluteFile()) == false) {
|
||||||
|
dirsToCheck.add(configuration.getSharedDownloadsDirectory().getAbsoluteFile());
|
||||||
|
}
|
||||||
|
if (configuration.getWoolCacheDirectory() != null && dirsToCheck.contains(configuration.getWoolCacheDirectory().getAbsoluteFile()) == false) {
|
||||||
|
dirsToCheck.add(configuration.getWoolCacheDirectory().getAbsoluteFile());
|
||||||
|
}
|
||||||
|
ListIterator<File> dirs = dirsToCheck.listIterator();
|
||||||
|
while (dirs.hasNext()) {
|
||||||
|
File dir = dirs.next();
|
||||||
|
dirs.remove();
|
||||||
|
dirsChecked.add(dir);
|
||||||
|
File[] fileList = dir.listFiles();
|
||||||
|
if (fileList == null) {
|
||||||
|
logs.add(f + "File list of " + dir + " is null");
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
for (File file : fileList) {
|
||||||
|
file = file.getAbsoluteFile();
|
||||||
|
//logs.add(f + file);
|
||||||
|
boolean canRead = file.canRead();
|
||||||
|
boolean canWrite = file.canWrite();
|
||||||
|
boolean isDir = file.isDirectory();
|
||||||
|
if (canRead == false) {
|
||||||
|
logs.add(f + "Can't read from " + file);
|
||||||
|
}
|
||||||
|
if (canWrite == false) {
|
||||||
|
logs.add(f + "Can't write to " + file);
|
||||||
|
}
|
||||||
|
if (canRead && canWrite && isDir) {
|
||||||
|
if (dirsChecked.contains(file)) {
|
||||||
|
logs.add(f + "Dir " + file + " already checked. Loop detected");
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
dirs.add(file);
|
||||||
|
}
|
||||||
|
long usableSpace = file.getUsableSpace();
|
||||||
|
if (usableSpace < 512 * 1024) {
|
||||||
|
logs.add(f + "Usable space is " + usableSpace + " for " + file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return logs;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,6 +80,8 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
private Configuration configuration;
|
private Configuration configuration;
|
||||||
private Log log;
|
private Log log;
|
||||||
|
|
||||||
|
private DirectoryManager directoryManager;
|
||||||
|
|
||||||
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
|
public Job(Configuration config_, Gui gui_, Log log_, String id_, String frame_, String path_, boolean use_gpu, String command_, String validationUrl_,
|
||||||
String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
|
String script_, DownloadItem projectDownload_, DownloadItem rendererDownload_, String name_, char[] password_, boolean synchronous_upload_,
|
||||||
String update_method_) {
|
String update_method_) {
|
||||||
@@ -94,6 +96,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
synchronousUpload = synchronous_upload_;
|
synchronousUpload = synchronous_upload_;
|
||||||
gui = gui_;
|
gui = gui_;
|
||||||
log = log_;
|
log = log_;
|
||||||
|
directoryManager = new DirectoryManager(configuration);
|
||||||
renderProcess = new RenderProcess(log_);
|
renderProcess = new RenderProcess(log_);
|
||||||
renderState = new RenderState();
|
renderState = new RenderState();
|
||||||
renderOutput = new RenderOutput();
|
renderOutput = new RenderOutput();
|
||||||
@@ -259,7 +262,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
catch (IOException e) {
|
catch (IOException e) {
|
||||||
StringWriter sw = new StringWriter();
|
StringWriter sw = new StringWriter();
|
||||||
e.printStackTrace(new PrintWriter(sw));
|
e.printStackTrace(new PrintWriter(sw));
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
||||||
@@ -293,7 +296,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
catch (IOException e) {
|
catch (IOException e) {
|
||||||
StringWriter sw = new StringWriter();
|
StringWriter sw = new StringWriter();
|
||||||
e.printStackTrace(new PrintWriter(sw));
|
e.printStackTrace(new PrintWriter(sw));
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
log.error("Job::render exception on script generation, will return UNKNOWN " + e + " stacktrace " + sw.toString());
|
||||||
@@ -474,7 +477,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
gui.updateTrayIcon(SHOW_BASE_ICON);
|
gui.updateTrayIcon(SHOW_BASE_ICON);
|
||||||
process.kill();
|
process.kill();
|
||||||
maybeCleanWorkingDir(error);
|
maybeCleanWorkingDir(error);
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -536,7 +539,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
}
|
}
|
||||||
StringWriter sw = new StringWriter();
|
StringWriter sw = new StringWriter();
|
||||||
err.printStackTrace(new PrintWriter(sw));
|
err.printStackTrace(new PrintWriter(sw));
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render exception(A) " + err + " stacktrace " + sw.toString());
|
log.error("Job::render exception(A) " + err + " stacktrace " + sw.toString());
|
||||||
@@ -591,7 +594,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (files.length == 0) {
|
if (files.length == 0) {
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render no picture file found (after finished render (filename_without_extension " + filename_without_extension + ")");
|
log.error("Job::render no picture file found (after finished render (filename_without_extension " + filename_without_extension + ")");
|
||||||
@@ -605,7 +608,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
}
|
}
|
||||||
File crash_file = new File(configuration.getWorkingDirectory() + File.separator + basename + ".crash.txt");
|
File crash_file = new File(configuration.getWorkingDirectory() + File.separator + basename + ".crash.txt");
|
||||||
if (crash_file.exists()) {
|
if (crash_file.exists()) {
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render crash file found => the renderer crashed");
|
log.error("Job::render crash file found => the renderer crashed");
|
||||||
@@ -614,7 +617,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (exit_value == 127 && process.getDuration() < 10) {
|
if (exit_value == 127 && process.getDuration() < 10) {
|
||||||
for (String logline : configuration.filesystemHealthCheck()) {
|
for (String logline : directoryManager.filesystemHealthCheck()) {
|
||||||
log.debug(logline);
|
log.debug(logline);
|
||||||
}
|
}
|
||||||
log.error("Job::render renderer returned 127 and took " + process.getDuration() + "s, some libraries may be missing");
|
log.error("Job::render renderer returned 127 and took " + process.getDuration() + "s, some libraries may be missing");
|
||||||
@@ -771,7 +774,7 @@ import static com.sheepit.client.RenderSettings.UPDATE_METHOD_BY_TILE;
|
|||||||
|| Type.ENGINE_NOT_AVAILABLE == error
|
|| Type.ENGINE_NOT_AVAILABLE == error
|
||||||
|| Type.DETECT_DEVICE_ERROR == error;
|
|| Type.DETECT_DEVICE_ERROR == error;
|
||||||
if (cleanup) {
|
if (cleanup) {
|
||||||
configuration.cleanWorkingDirectory();
|
directoryManager.cleanWorkingDirectory();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ public class Server extends Thread {
|
|||||||
private Configuration user_config;
|
private Configuration user_config;
|
||||||
private Client client;
|
private Client client;
|
||||||
private Log log;
|
private Log log;
|
||||||
|
private DirectoryManager directoryManager;
|
||||||
private long lastRequestTime;
|
private long lastRequestTime;
|
||||||
private int keepmealive_duration; // time in ms
|
private int keepmealive_duration; // time in ms
|
||||||
|
|
||||||
@@ -107,6 +108,7 @@ public class Server extends Thread {
|
|||||||
this.user_config = user_config_;
|
this.user_config = user_config_;
|
||||||
this.client = client_;
|
this.client = client_;
|
||||||
this.log = Log.getInstance(this.user_config);
|
this.log = Log.getInstance(this.user_config);
|
||||||
|
this.directoryManager = new DirectoryManager(this.user_config);
|
||||||
this.lastRequestTime = 0;
|
this.lastRequestTime = 0;
|
||||||
this.keepmealive_duration = 15 * 60 * 1000; // default 15min
|
this.keepmealive_duration = 15 * 60 * 1000; // default 15min
|
||||||
|
|
||||||
@@ -690,7 +692,7 @@ public class Server extends Thread {
|
|||||||
|
|
||||||
private String generateXMLForMD5cache() {
|
private String generateXMLForMD5cache() {
|
||||||
List<FileMD5> md5s = new ArrayList<>();
|
List<FileMD5> md5s = new ArrayList<>();
|
||||||
for (File local_file : this.user_config.getLocalCacheFiles()) {
|
for (File local_file : this.directoryManager.getLocalCacheFiles()) {
|
||||||
try {
|
try {
|
||||||
String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase();
|
String extension = local_file.getName().substring(local_file.getName().lastIndexOf('.')).toLowerCase();
|
||||||
String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length());
|
String name = local_file.getName().substring(0, local_file.getName().length() - 1 * extension.length());
|
||||||
|
|||||||
@@ -526,6 +526,7 @@ public class SettingsLoader {
|
|||||||
|
|
||||||
if (config.isUserHasSpecifiedACacheDir() == false && cacheDir != null) {
|
if (config.isUserHasSpecifiedACacheDir() == false && cacheDir != null) {
|
||||||
config.setCacheDir(new File(cacheDir.getValue()));
|
config.setCacheDir(new File(cacheDir.getValue()));
|
||||||
|
(new DirectoryManager(config)).createCacheDir();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.getUIType() == null && ui != null) {
|
if (config.getUIType() == null && ui != null) {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@
|
|||||||
|
|
||||||
package com.sheepit.client.standalone;
|
package com.sheepit.client.standalone;
|
||||||
|
|
||||||
|
import com.sheepit.client.DirectoryManager;
|
||||||
import com.sheepit.client.hardware.gpu.hip.HIP;
|
import com.sheepit.client.hardware.gpu.hip.HIP;
|
||||||
import org.kohsuke.args4j.CmdLineException;
|
import org.kohsuke.args4j.CmdLineException;
|
||||||
import org.kohsuke.args4j.CmdLineParser;
|
import org.kohsuke.args4j.CmdLineParser;
|
||||||
@@ -149,7 +150,6 @@ public class Worker {
|
|||||||
config.setDetectGPUs(!no_gpu_detection);
|
config.setDetectGPUs(!no_gpu_detection);
|
||||||
config.setIncompatibleProcess(incompatibleProcess);
|
config.setIncompatibleProcess(incompatibleProcess);
|
||||||
|
|
||||||
|
|
||||||
if (sharedDownloadsDir != null) {
|
if (sharedDownloadsDir != null) {
|
||||||
File dir = new File(sharedDownloadsDir);
|
File dir = new File(sharedDownloadsDir);
|
||||||
if (dir.exists() == false && dir.mkdirs()) {
|
if (dir.exists() == false && dir.mkdirs()) {
|
||||||
@@ -176,7 +176,10 @@ public class Worker {
|
|||||||
File a_dir = new File(cache_dir);
|
File a_dir = new File(cache_dir);
|
||||||
a_dir.mkdirs();
|
a_dir.mkdirs();
|
||||||
if (a_dir.isDirectory() && a_dir.canWrite()) {
|
if (a_dir.isDirectory() && a_dir.canWrite()) {
|
||||||
|
DirectoryManager directoryManager = new DirectoryManager(config);
|
||||||
|
|
||||||
config.setCacheDir(a_dir);
|
config.setCacheDir(a_dir);
|
||||||
|
directoryManager.createCacheDir();
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
System.err.println("ERROR: The entered cache path is either not a directory or is not writable!");
|
System.err.println("ERROR: The entered cache path is either not a directory or is not writable!");
|
||||||
|
|||||||
@@ -60,6 +60,7 @@ import com.formdev.flatlaf.FlatDarkLaf;
|
|||||||
|
|
||||||
import com.sheepit.client.Configuration;
|
import com.sheepit.client.Configuration;
|
||||||
import com.sheepit.client.Configuration.ComputeType;
|
import com.sheepit.client.Configuration.ComputeType;
|
||||||
|
import com.sheepit.client.DirectoryManager;
|
||||||
import com.sheepit.client.hardware.cpu.CPU;
|
import com.sheepit.client.hardware.cpu.CPU;
|
||||||
import com.sheepit.client.hardware.gpu.GPU;
|
import com.sheepit.client.hardware.gpu.GPU;
|
||||||
import com.sheepit.client.hardware.gpu.GPUDevice;
|
import com.sheepit.client.hardware.gpu.GPUDevice;
|
||||||
@@ -707,6 +708,7 @@ public class Settings implements Activity {
|
|||||||
File fromConfig = config.getWorkingDirectory();
|
File fromConfig = config.getWorkingDirectory();
|
||||||
if (fromConfig != null && fromConfig.getAbsolutePath().equals(cacheDir.getAbsolutePath()) == false) {
|
if (fromConfig != null && fromConfig.getAbsolutePath().equals(cacheDir.getAbsolutePath()) == false) {
|
||||||
config.setCacheDir(cacheDir);
|
config.setCacheDir(cacheDir);
|
||||||
|
(new DirectoryManager(config)).createCacheDir();
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// do nothing because the directory is the same as before
|
// do nothing because the directory is the same as before
|
||||||
|
|||||||
Reference in New Issue
Block a user