diff --git a/src/main/java/com/sheepit/client/Client.java b/src/main/java/com/sheepit/client/Client.java index 84f91ad..6d4c866 100644 --- a/src/main/java/com/sheepit/client/Client.java +++ b/src/main/java/com/sheepit/client/Client.java @@ -39,6 +39,7 @@ import java.util.TimerTask; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -97,6 +98,7 @@ import okhttp3.HttpUrl; private boolean isValidatingJob; private long startTime; private boolean sessionStarted; + private CompletableFuture directoryInit; private boolean disableErrorSending; private boolean running; @@ -108,7 +110,7 @@ import okhttp3.HttpUrl; private long uploadQueueVolume; private int noJobRetryIter; - public Client(Gui gui, Configuration configuration, String url) { + public Client(Gui gui, Configuration configuration, String url, CompletableFuture directoryInit) { this.configuration = configuration; this.serverRequest = new ServerRequest(url, this.configuration, this); this.log = Log.getInstance(); @@ -130,6 +132,7 @@ import okhttp3.HttpUrl; this.noJobRetryIter = 0; this.sessionStarted = false; + this.directoryInit = directoryInit; } @Override public String toString() { @@ -154,6 +157,8 @@ import okhttp3.HttpUrl; step = this.log.newCheckPoint(); this.gui.status("Starting"); + // Wait for the cache to finish setup + this.directoryInit.join(); Error.Type ret; ret = this.serverRequest.getConfiguration(); diff --git a/src/main/java/com/sheepit/client/config/DirectoryCleaner.java b/src/main/java/com/sheepit/client/config/DirectoryCleaner.java new file mode 100644 index 0000000..0ca282d --- /dev/null +++ b/src/main/java/com/sheepit/client/config/DirectoryCleaner.java @@ -0,0 +1,83 @@ +package com.sheepit.client.config; + +import com.sheepit.client.logger.Log; +import com.sheepit.client.utils.Utils; +import lombok.AllArgsConstructor; +import org.apache.commons.io.FilenameUtils; + +import java.io.File; + +@AllArgsConstructor +public class DirectoryCleaner implements Runnable { + + public static final String FILE_SEPARATOR = " file: "; + static Log log = Log.getInstance(); + private File directory; + + + @Override public void run() { + log.debug("DirectoryCleaner initialized. Cleaning " + directory.getName()); + cleanDirectory(); + } + + /** + * Cleans a directory and removes files in it from the md5 cache + */ + private void cleanDirectory() { + if (directory == null) { + return; + } + final String LOG_PREFIX = "DirectoryCleaner::cleanDirectory:"+ directory.getName(); + File[] files = directory.listFiles(); + int removedFilesCount = 0; + if (files != null) { + log.debug(LOG_PREFIX + " found files in directory " + files.length); + for (File file : files) { + if (file.isDirectory()) { + Utils.delete(file); + } + else { + try { + String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); + String name = FilenameUtils.removeExtension(file.getName()); + if ("wool".equals(extension)) { + String md5Local = Utils.md5(file.getAbsolutePath()); + // Check if .wool file is corrupt + if (md5Local.equals(name) == false) { + if (file.delete()) { + removedFilesCount++; + } + else { + String baseMessage = " failed to delete .wool file that is corrupt"; + String message = LOG_PREFIX + baseMessage + " file: " + file.getName(); + log.debug(message); + } + } + } + else { + if (file.delete()) { + removedFilesCount++; + } + else { + String baseMessage = LOG_PREFIX + " failed to delete non-wool file that is not present in cache"; + String message = baseMessage + FILE_SEPARATOR + file.getName(); + log.debug(message); + } + } + } + catch (IllegalArgumentException e) { // because the file does not have a . in its path + if (file.delete()) { + removedFilesCount++; + } + else { + String baseMessage = LOG_PREFIX + " failed to delete file with no extension"; + String message = baseMessage + FILE_SEPARATOR + file.getName(); + log.debug(message); + } + } + } + } + } + log.debug(LOG_PREFIX + " successfully removed " + removedFilesCount + " from " + directory.getName()); + } +} diff --git a/src/main/java/com/sheepit/client/config/DirectoryManager.java b/src/main/java/com/sheepit/client/config/DirectoryManager.java index fe3536f..3f44ff3 100644 --- a/src/main/java/com/sheepit/client/config/DirectoryManager.java +++ b/src/main/java/com/sheepit/client/config/DirectoryManager.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; +import java.util.concurrent.CompletableFuture; @AllArgsConstructor public class DirectoryManager { @@ -62,6 +63,8 @@ public class DirectoryManager { return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk)); } + private static final Log log = Log.getInstance(); + private boolean copyFileFromSharedToCache(String source, String target) { Path existingArchivePath = Paths.get(source); Path targetArchivePath = Paths.get(target); @@ -82,12 +85,12 @@ public class DirectoryManager { | SecurityException // user is not allowed to create hard-links ignore) { // Creating hardlinks might not be supported on some filesystems - Log.getInstance().debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); + log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath); Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING); } } catch (IOException e) { - Log.getInstance().error("Error while copying " + source + " from shared downloads directory to working dir"); + log.error("Error while copying " + source + " from shared downloads directory to working dir"); return false; } @@ -107,10 +110,14 @@ public class DirectoryManager { this.configuration.getSharedDownloadsDirectory().mkdirs(); if (this.configuration.getSharedDownloadsDirectory().exists() == false) { - System.err.println("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath()); + log.error("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath()); } } } + + public CompletableFuture createCacheDirAsync() { + return CompletableFuture.runAsync(this::createCacheDir); + } /** * Cleans working directory and also deletes it if the user hasn't specified a cache directory @@ -128,51 +135,8 @@ public class DirectoryManager { * Deletes the working and storage directories */ public void cleanWorkingDirectory() { - this.cleanDirectory(this.configuration.getWorkingDirectory()); - this.cleanDirectory(this.configuration.getWoolCacheDirectory()); - } - - /** - * Cleans a directory and removes files in it from the md5 cache - * @param dir representing the directory to be cleaned - * @return false if the dir null, true otherwise - */ - private boolean cleanDirectory(File dir) { - if (dir == null) { - return false; - } - - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - Utils.delete(file); - } - else { - try { - String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); - String name = FilenameUtils.removeExtension(file.getName()); - if ("wool".equals(extension)) { - // check if the md5 of the file is ok - String md5_local = Utils.md5(file.getAbsolutePath()); - - if (md5_local.equals(name) == false) { - file.delete(); - } - - // TODO: remove old one - } - else { - file.delete(); - } - } - catch (IllegalArgumentException e) { // because the file does not have an . in his path - file.delete(); - } - } - } - } - return true; + new DirectoryCleaner(this.configuration.getWorkingDirectory()).run(); + new DirectoryCleaner(this.configuration.getWoolCacheDirectory()).run(); } /** @@ -180,7 +144,7 @@ public class DirectoryManager { * working, storage, and shared downloads directories as long as they are not null */ public List getLocalCacheFiles() { - List files_local = new LinkedList(); + List filesLocal = new LinkedList(); List files = new LinkedList(); if (this.configuration.getWorkingDirectory() != null) { File[] filesInDirectory = this.configuration.getWorkingDirectory().listFiles(); @@ -208,10 +172,10 @@ public class DirectoryManager { String name = FilenameUtils.removeExtension(file.getName()); if ("wool".equals(extension)) { // check if the md5 of the file is ok - String md5_local = Utils.md5(file.getAbsolutePath()); + String md5Local = Utils.md5(file.getAbsolutePath()); - if (md5_local.equals(name)) { - files_local.add(file); + if (md5Local.equals(name)) { + filesLocal.add(file); } } } @@ -220,7 +184,7 @@ public class DirectoryManager { } } - return files_local; + return filesLocal; } /** diff --git a/src/main/java/com/sheepit/client/config/SettingsLoader.java b/src/main/java/com/sheepit/client/config/SettingsLoader.java index 1366377..cf61d85 100644 --- a/src/main/java/com/sheepit/client/config/SettingsLoader.java +++ b/src/main/java/com/sheepit/client/config/SettingsLoader.java @@ -523,7 +523,6 @@ public class SettingsLoader { if (config.isUserHasSpecifiedACacheDir() == false && cacheDir != null) { config.setCacheDir(new File(cacheDir.getValue())); - (new DirectoryManager(config)).createCacheDir(); } if (config.getUIType() == null && ui != null) { diff --git a/src/main/java/com/sheepit/client/main/Worker.java b/src/main/java/com/sheepit/client/main/Worker.java index 3117f2d..d3e0fab 100644 --- a/src/main/java/com/sheepit/client/main/Worker.java +++ b/src/main/java/com/sheepit/client/main/Worker.java @@ -40,6 +40,7 @@ import java.time.temporal.ChronoUnit; import java.util.Calendar; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -476,9 +477,17 @@ public class Worker { String configFilePath = config.getConfigFilePath() != null ? config.getConfigFilePath() : OS.getOS().getDefaultConfigFilePath(); config.setLogDirectory(log_dir != null ? log_dir : (new File (configFilePath).getParent())); - (new DirectoryManager(config)).createCacheDir(); - Log.setInstance(config); + boolean shouldCreateCache = config.isUserHasSpecifiedACacheDir() == false && cache_dir != null; + CompletableFuture directoryInit; + if (shouldCreateCache) { + directoryInit = (new DirectoryManager(config)).createCacheDirAsync(); + } + else { + directoryInit = CompletableFuture.completedFuture(null); + } + + Log.setInstance(config); Log.getInstance().debug("client version " + Configuration.jarVersion); // Hostname change will overwrite the existing one (default or read from configuration file) but changes will be lost when the client closes @@ -524,7 +533,7 @@ public class Worker { ((GuiSwing) gui).setSettingsLoader(settingsLoader); break; } - Client cli = new Client(gui, config, server); + Client cli = new Client(gui, config, server, directoryInit); gui.setClient(cli); ShutdownHook hook = new ShutdownHook(cli); hook.attachShutDownHook(); diff --git a/src/main/java/com/sheepit/client/utils/Md5.java b/src/main/java/com/sheepit/client/utils/Md5.java index b288428..9986dd9 100644 --- a/src/main/java/com/sheepit/client/utils/Md5.java +++ b/src/main/java/com/sheepit/client/utils/Md5.java @@ -27,15 +27,19 @@ import java.nio.file.Paths; import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; -public class Md5 { - private static Map cache = new HashMap<>(); +final class Md5 { + + // Maps the file path to a MD5 hash of the file + static final Map cache = new ConcurrentHashMap<>(); // TODO: to avoid memory increase, check if files are deleted - public String get(String path) { + private Md5() {} + + static synchronized String get(String path) { String key = getUniqueKey(path); if (cache.containsKey(key) == false) { generate(path); @@ -43,19 +47,20 @@ public class Md5 { return cache.get(key); } - private void generate(String path) { + static synchronized void generate(String path) { String key = getUniqueKey(path); try { MessageDigest md = MessageDigest.getInstance("MD5"); - InputStream is = Files.newInputStream(Paths.get(path)); - DigestInputStream dis = new DigestInputStream(is, md); - byte[] buffer = new byte[8192]; - while (dis.read(buffer) > 0) { - // process the entire file + + String data; + try (InputStream is = Files.newInputStream(Paths.get(path)); + DigestInputStream dis = new DigestInputStream(is, md)) { + byte[] buffer = new byte[256 * 1024]; // 256 KiB buffer + while (dis.read(buffer) > 0) { + // process the entire file + } + data = Utils.convertBinaryToHex(md.digest()); } - String data = Utils.convertBinaryToHex(md.digest()); - dis.close(); - is.close(); cache.put(key, data); } catch (NoSuchAlgorithmException | IOException e) { @@ -63,7 +68,7 @@ public class Md5 { } } - private String getUniqueKey(String path) { + static synchronized String getUniqueKey(String path) { File file = new File(path); return Long.toString(file.lastModified()) + '_' + path; } diff --git a/src/main/java/com/sheepit/client/utils/Utils.java b/src/main/java/com/sheepit/client/utils/Utils.java index 7eaed3f..24a9c3b 100644 --- a/src/main/java/com/sheepit/client/utils/Utils.java +++ b/src/main/java/com/sheepit/client/utils/Utils.java @@ -22,16 +22,12 @@ package com.sheepit.client.utils; import com.sheepit.client.zip.ChunkInputStream; import com.sheepit.client.logger.Log; import com.sheepit.client.zip.UnzipUtils; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.PrintWriter; -import java.io.StringWriter; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; @@ -63,36 +59,6 @@ public class Utils { mimeTypes.put(".exr", "image/x-exr"); } - /** - * Extracts (and optionally decrypts) contents of a given zip file to a target directory - * - * @param zipFileName_ Path to the zipfiles - * @param destinationDirectory Path to the target directory where files will be extracted to - * @param password Optional password for decrypting the zip archive which will only be used if it's not null and if the zip is truly encrypted - * @param log The SheepIt Debug log where log messages might be logged into - * @return A status code as an integer, -1 if it encounters a ZipException, 0 otherwise - */ - public static int unzipFileIntoDirectory(String zipFileName_, String destinationDirectory, char[] password, Log log) { - try { - ZipFile zipFile = new ZipFile(zipFileName_); -// unzipParameters.setIgnoreDateTimeAttributes(true); - - if (password != null && zipFile.isEncrypted()) { - zipFile.setPassword(password); - } -// zipFile.extractAll(destinationDirectory, unzipParameters); - zipFile.extractAll(destinationDirectory); - } - catch (ZipException e) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - log.debug("Utils::unzipFileIntoDirectory(" + zipFileName_ + "," + destinationDirectory + ") exception " + e + " stacktrace: " + sw.toString()); - return -1; - } - return 0; - } - /** * Takes the list of zip file chunks and combines them to a zip archive in memory to * extract (and optionally decrypt) the contents to the target directory @@ -125,8 +91,7 @@ public class Utils { * @return the string The MD5 checksum */ public static String md5(String path_of_file_) { - Md5 md5 = new Md5(); - return md5.get(path_of_file_); + return Md5.get(path_of_file_); } /**