Improve MD5 and startup speed

src/main/java/com/sheepit/client/Client.java

@@ -39,6 +39,7 @@ import java.util.TimerTask;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -97,6 +98,7 @@ import okhttp3.HttpUrl;
 	private boolean isValidatingJob;
 	private long startTime;
 	private boolean sessionStarted;
+	private CompletableFuture<Void> directoryInit;
 	
 	private boolean disableErrorSending;
 	private boolean running;
@@ -108,7 +110,7 @@ import okhttp3.HttpUrl;
 	private long uploadQueueVolume;
 	private int noJobRetryIter;
 	
-	public Client(Gui gui, Configuration configuration, String url) {
+	public Client(Gui gui, Configuration configuration, String url, CompletableFuture<Void> directoryInit) {
 		this.configuration = configuration;
 		this.serverRequest = new ServerRequest(url, this.configuration, this);
 		this.log = Log.getInstance();
@@ -130,6 +132,7 @@ import okhttp3.HttpUrl;
 		this.noJobRetryIter = 0;
 		
 		this.sessionStarted = false;
+		this.directoryInit = directoryInit;
 	}
 	
 	@Override public String toString() {
@@ -154,6 +157,8 @@ import okhttp3.HttpUrl;
 			step = this.log.newCheckPoint();
 			this.gui.status("Starting");
 			
+			// Wait for the cache to finish setup
+			this.directoryInit.join();
 			Error.Type ret;
 			ret = this.serverRequest.getConfiguration();
 			

src/main/java/com/sheepit/client/config/DirectoryCleaner.java

@@ -0,0 +1,83 @@
+package com.sheepit.client.config;
+
+import com.sheepit.client.logger.Log;
+import com.sheepit.client.utils.Utils;
+import lombok.AllArgsConstructor;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+
+@AllArgsConstructor
+public class DirectoryCleaner implements Runnable {
+	
+	public static final String FILE_SEPARATOR = " file: ";
+	static Log log = Log.getInstance();
+	private File directory;
+
+	
+	@Override public void run() {
+		log.debug("DirectoryCleaner initialized. Cleaning " + directory.getName());
+		cleanDirectory();
+	}
+	
+	/**
+	 * Cleans a directory and removes files in it from the md5 cache
+	 */
+	private void cleanDirectory() {
+		if (directory == null) {
+			return;
+		}
+		final String LOG_PREFIX = "DirectoryCleaner::cleanDirectory:"+ directory.getName();
+		File[] files = directory.listFiles();
+		int removedFilesCount = 0;
+		if (files != null) {
+			log.debug(LOG_PREFIX + " found files in directory " + files.length);
+			for (File file : files) {
+				if (file.isDirectory()) {
+					Utils.delete(file);
+				}
+				else {
+					try {
+						String extension = FilenameUtils.getExtension(file.getName()).toLowerCase();
+						String name = FilenameUtils.removeExtension(file.getName());
+						if ("wool".equals(extension)) {
+							String md5Local = Utils.md5(file.getAbsolutePath());
+							// Check if .wool file is corrupt
+							if (md5Local.equals(name) == false) {
+								if (file.delete()) {
+									removedFilesCount++;
+								}
+								else {
+									String baseMessage = " failed to delete .wool file that is corrupt";
+									String message = LOG_PREFIX + baseMessage + " file: " + file.getName();
+									log.debug(message);
+								}
+							}
+						}
+						else {
+							if (file.delete()) {
+								removedFilesCount++;
+							}
+							else {
+								String baseMessage = LOG_PREFIX + " failed to delete non-wool file that is not present in cache";
+								String message = baseMessage + FILE_SEPARATOR + file.getName();
+								log.debug(message);
+							}
+						}
+					}
+					catch (IllegalArgumentException e) { // because the file does not have a . in its path
+						if (file.delete()) {
+							removedFilesCount++;
+						}
+						else {
+							String baseMessage = LOG_PREFIX + " failed to delete file with no extension";
+							String message = baseMessage + FILE_SEPARATOR + file.getName();
+							log.debug(message);
+						}
+					}
+				}
+			}
+		}
+		log.debug(LOG_PREFIX + " successfully removed " + removedFilesCount + " from " + directory.getName());
+	}
+}

src/main/java/com/sheepit/client/config/DirectoryManager.java

@@ -37,6 +37,7 @@ import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.ListIterator;
+import java.util.concurrent.CompletableFuture;
 
 @AllArgsConstructor
 public class DirectoryManager {
@@ -62,6 +63,8 @@ public class DirectoryManager {
 		return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk));
 	}
 	
+	private static final Log log = Log.getInstance();
+	
 	private boolean copyFileFromSharedToCache(String source, String target) {
 		Path existingArchivePath = Paths.get(source);
 		Path targetArchivePath = Paths.get(target);
@@ -82,12 +85,12 @@ public class DirectoryManager {
 				   | SecurityException         // user is not allowed to create hard-links
 				ignore) {
 				// Creating hardlinks might not be supported on some filesystems
-				Log.getInstance().debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
+				log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
 				Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
 			}
 		}
 		catch (IOException e) {
-			Log.getInstance().error("Error while copying " + source + " from shared downloads directory to working dir");
+			log.error("Error while copying " + source + " from shared downloads directory to working dir");
 			return false;
 		}
 		
@@ -107,11 +110,15 @@ public class DirectoryManager {
 			this.configuration.getSharedDownloadsDirectory().mkdirs();
 
 			if (this.configuration.getSharedDownloadsDirectory().exists() == false) {
-				System.err.println("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath());
+				log.error("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath());
 			}
 		}
 	}
 
+	public CompletableFuture<Void> createCacheDirAsync() {
+		return CompletableFuture.runAsync(this::createCacheDir);
+	}
+	
 	/**
 	 * Cleans working directory and also deletes it if the user hasn't specified a cache directory
 	 */
@@ -128,51 +135,8 @@ public class DirectoryManager {
 	 * Deletes the working and storage directories
 	 */
 	public void cleanWorkingDirectory() {
-		this.cleanDirectory(this.configuration.getWorkingDirectory());
+		new DirectoryCleaner(this.configuration.getWorkingDirectory()).run();
-		this.cleanDirectory(this.configuration.getWoolCacheDirectory());
+		new DirectoryCleaner(this.configuration.getWoolCacheDirectory()).run();
-	}
-	
-	/**
-	 * Cleans a directory and removes files in it from the md5 cache
-	 * @param dir representing the directory to be cleaned
-	 * @return false if the dir null, true otherwise
-	 */
-	private boolean cleanDirectory(File dir) {
-		if (dir == null) {
-			return false;
-		}
-		
-		File[] files = dir.listFiles();
-		if (files != null) {
-			for (File file : files) {
-				if (file.isDirectory()) {
-					Utils.delete(file);
-				}
-				else {
-					try {
-						String extension = FilenameUtils.getExtension(file.getName()).toLowerCase();
-						String name = FilenameUtils.removeExtension(file.getName());
-						if ("wool".equals(extension)) {
-							// check if the md5 of the file is ok
-							String md5_local = Utils.md5(file.getAbsolutePath());
-							
-							if (md5_local.equals(name) == false) {
-								file.delete();
-							}
-							
-							// TODO: remove old one
-						}
-						else {
-							file.delete();
-						}
-					}
-					catch (IllegalArgumentException e) { // because the file does not have an . in his path
-						file.delete();
-					}
-				}
-			}
-		}
-		return true;
 	}
 	
 	/**
@@ -180,7 +144,7 @@ public class DirectoryManager {
 	 * working, storage, and shared downloads directories as long as they are not null
 	 */
 	public List<File> getLocalCacheFiles() {
-		List<File> files_local = new LinkedList<File>();
+		List<File> filesLocal = new LinkedList<File>();
 		List<File> files = new LinkedList<File>();
 		if (this.configuration.getWorkingDirectory() != null) {
 			File[] filesInDirectory = this.configuration.getWorkingDirectory().listFiles();
@@ -208,10 +172,10 @@ public class DirectoryManager {
 					String name = FilenameUtils.removeExtension(file.getName());
 					if ("wool".equals(extension)) {
 						// check if the md5 of the file is ok
-						String md5_local = Utils.md5(file.getAbsolutePath());
+						String md5Local = Utils.md5(file.getAbsolutePath());
 						
-						if (md5_local.equals(name)) {
+						if (md5Local.equals(name)) {
-							files_local.add(file);
+							filesLocal.add(file);
 						}
 					}
 				}
@@ -220,7 +184,7 @@ public class DirectoryManager {
 			}
 		}
 		
-		return files_local;
+		return filesLocal;
 	}
 	
 	/**

src/main/java/com/sheepit/client/config/SettingsLoader.java

@@ -523,7 +523,6 @@ public class SettingsLoader {
 		
 		if (config.isUserHasSpecifiedACacheDir() == false && cacheDir != null) {
 			config.setCacheDir(new File(cacheDir.getValue()));
-			(new DirectoryManager(config)).createCacheDir();
 		}
 		
 		if (config.getUIType() == null && ui != null) {

src/main/java/com/sheepit/client/main/Worker.java

@@ -40,6 +40,7 @@ import java.time.temporal.ChronoUnit;
 import java.util.Calendar;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -476,9 +477,17 @@ public class Worker {
 		String configFilePath = config.getConfigFilePath() != null ? config.getConfigFilePath() : OS.getOS().getDefaultConfigFilePath();
 		config.setLogDirectory(log_dir != null ? log_dir : (new File (configFilePath).getParent()));
 		
-		(new DirectoryManager(config)).createCacheDir();
+		boolean shouldCreateCache = config.isUserHasSpecifiedACacheDir() == false && cache_dir != null;
-		Log.setInstance(config);
+		CompletableFuture<Void> directoryInit;
 		
+		if (shouldCreateCache) {
+			directoryInit = (new DirectoryManager(config)).createCacheDirAsync();
+		}
+		else {
+			directoryInit = CompletableFuture.completedFuture(null);
+		}
+		
+		Log.setInstance(config);
 		Log.getInstance().debug("client version " + Configuration.jarVersion);
 		
 		// Hostname change will overwrite the existing one (default or read from configuration file) but changes will be lost when the client closes
@@ -524,7 +533,7 @@ public class Worker {
 				((GuiSwing) gui).setSettingsLoader(settingsLoader);
 				break;
 		}
-		Client cli = new Client(gui, config, server);
+		Client cli = new Client(gui, config, server, directoryInit);
 		gui.setClient(cli);
 		ShutdownHook hook = new ShutdownHook(cli);
 		hook.attachShutDownHook();

src/main/java/com/sheepit/client/utils/Md5.java

@@ -27,15 +27,19 @@ import java.nio.file.Paths;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
-public class Md5 {
+final class Md5 {
-	private static Map<String, String> cache = new HashMap<>();
+	
+	// Maps the file path to a MD5 hash of the file
+	static final Map<String, String> cache = new ConcurrentHashMap<>();
 	
 	// TODO: to avoid memory increase, check if files are deleted
 	
-	public String get(String path) {
+	private Md5() {}
+	
+	static synchronized String get(String path) {
 		String key = getUniqueKey(path);
 		if (cache.containsKey(key) == false) {
 			generate(path);
@@ -43,19 +47,20 @@ public class Md5 {
 		return cache.get(key);
 	}
 	
-	private void generate(String path) {
+	static synchronized void generate(String path) {
 		String key = getUniqueKey(path);
 		try {
 			MessageDigest md = MessageDigest.getInstance("MD5");
-			InputStream is = Files.newInputStream(Paths.get(path));
+			
-			DigestInputStream dis = new DigestInputStream(is, md);
+			String data;
-			byte[] buffer = new byte[8192];
+			try (InputStream is = Files.newInputStream(Paths.get(path));
-			while (dis.read(buffer) > 0) {
+				DigestInputStream dis = new DigestInputStream(is, md)) {
-				// process the entire file
+				byte[] buffer = new byte[256 * 1024]; // 256 KiB buffer
+				while (dis.read(buffer) > 0) {
+					// process the entire file
+				}
+				data = Utils.convertBinaryToHex(md.digest());
 			}
-			String data = Utils.convertBinaryToHex(md.digest());
-			dis.close();
-			is.close();
 			cache.put(key, data);
 		}
 		catch (NoSuchAlgorithmException | IOException e) {
@@ -63,7 +68,7 @@ public class Md5 {
 		}
 	}
 	
-	private String getUniqueKey(String path) {
+	 static synchronized String getUniqueKey(String path) {
 		File file = new File(path);
 		return Long.toString(file.lastModified()) + '_' + path;
 	}

src/main/java/com/sheepit/client/utils/Utils.java

@@ -22,16 +22,12 @@ package com.sheepit.client.utils;
 import com.sheepit.client.zip.ChunkInputStream;
 import com.sheepit.client.logger.Log;
 import com.sheepit.client.zip.UnzipUtils;
-import net.lingala.zip4j.ZipFile;
-import net.lingala.zip4j.exception.ZipException;
 
 import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -63,36 +59,6 @@ public class Utils {
 		mimeTypes.put(".exr", "image/x-exr");
 	}
 	
-	/**
-	 * Extracts (and optionally decrypts) contents of a given zip file to a target directory
-	 *
-	 * @param zipFileName_ Path to the zipfiles
-	 * @param destinationDirectory Path to the target directory where files will be extracted to
-	 * @param password Optional password for decrypting the zip archive which will only be used if it's not null and if the zip is truly encrypted
-	 * @param log The SheepIt Debug log where log messages might be logged into
-	 * @return A status code as an integer, -1 if it encounters a ZipException, 0 otherwise
-	 */
-	public static int unzipFileIntoDirectory(String zipFileName_, String destinationDirectory, char[] password, Log log) {
-		try {
-			ZipFile zipFile = new ZipFile(zipFileName_);
-//			unzipParameters.setIgnoreDateTimeAttributes(true);
-			
-			if (password != null && zipFile.isEncrypted()) {
-				zipFile.setPassword(password);
-			}
-//			zipFile.extractAll(destinationDirectory, unzipParameters);
-			zipFile.extractAll(destinationDirectory);
-		}
-		catch (ZipException e) {
-			StringWriter sw = new StringWriter();
-			PrintWriter pw = new PrintWriter(sw);
-			e.printStackTrace(pw);
-			log.debug("Utils::unzipFileIntoDirectory(" + zipFileName_ + "," + destinationDirectory + ") exception " + e + " stacktrace: " + sw.toString());
-			return -1;
-		}
-		return 0;
-	}
-	
 	/**
 	 * Takes the list of zip file chunks and combines them to a zip archive in memory to
 	 * extract (and optionally decrypt) the contents to the target directory
@@ -125,8 +91,7 @@ public class Utils {
 	 * @return the string The MD5 checksum
 	 */
 	public static String md5(String path_of_file_) {
-		Md5 md5 = new Md5();
+		return Md5.get(path_of_file_);
-		return md5.get(path_of_file_);
 	}
 	
 	/**