Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve MD5 and startup speed

Browse Source
This commit is contained in:
Luke
2025-08-17 07:27:14 +00:00
committed by Laurent Clouet
parent ccdfc5d3f4
commit a8437ccef6
7 changed files with 138 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/com/sheepit/client/Client.java
View File

@@ -39,6 +39,7 @@ import java.util.TimerTask;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -97,6 +98,7 @@ import okhttp3.HttpUrl;
private boolean isValidatingJob;
private long startTime;
private boolean sessionStarted;
private CompletableFuture<Void> directoryInit;
private boolean disableErrorSending;
private boolean running;
@@ -108,7 +110,7 @@ import okhttp3.HttpUrl;
private long uploadQueueVolume;
private int noJobRetryIter;
public Client(Gui gui, Configuration configuration, String url) {
public Client(Gui gui, Configuration configuration, String url, CompletableFuture<Void> directoryInit) {
this.configuration = configuration;
this.serverRequest = new ServerRequest(url, this.configuration, this);
this.log = Log.getInstance();
@@ -130,6 +132,7 @@ import okhttp3.HttpUrl;
this.noJobRetryIter = 0;
this.sessionStarted = false;
this.directoryInit = directoryInit;
}
@Override public String toString() {
@@ -154,6 +157,8 @@ import okhttp3.HttpUrl;
step = this.log.newCheckPoint();
this.gui.status("Starting");
// Wait for the cache to finish setup
this.directoryInit.join();
Error.Type ret;
ret = this.serverRequest.getConfiguration();

83
src/main/java/com/sheepit/client/config/DirectoryCleaner.java Normal file
View File

@@ -0,0 +1,83 @@
package com.sheepit.client.config;
import com.sheepit.client.logger.Log;
import com.sheepit.client.utils.Utils;
import lombok.AllArgsConstructor;
import org.apache.commons.io.FilenameUtils;
import java.io.File;
@AllArgsConstructor
public class DirectoryCleaner implements Runnable {
public static final String FILE_SEPARATOR = " file: ";
static Log log = Log.getInstance();
private File directory;
@Override public void run() {
log.debug("DirectoryCleaner initialized. Cleaning " + directory.getName());
cleanDirectory();
}
/**
* Cleans a directory and removes files in it from the md5 cache
*/
private void cleanDirectory() {
if (directory == null) {
return;
}
final String LOG_PREFIX = "DirectoryCleaner::cleanDirectory:"+ directory.getName();
File[] files = directory.listFiles();
int removedFilesCount = 0;
if (files != null) {
log.debug(LOG_PREFIX + " found files in directory " + files.length);
for (File file : files) {
if (file.isDirectory()) {
Utils.delete(file);
}
else {
try {
String extension = FilenameUtils.getExtension(file.getName()).toLowerCase();
String name = FilenameUtils.removeExtension(file.getName());
if ("wool".equals(extension)) {
String md5Local = Utils.md5(file.getAbsolutePath());
// Check if .wool file is corrupt
if (md5Local.equals(name) == false) {
if (file.delete()) {
removedFilesCount++;
}
else {
String baseMessage = " failed to delete .wool file that is corrupt";
String message = LOG_PREFIX + baseMessage + " file: " + file.getName();
log.debug(message);
}
}
}
else {
if (file.delete()) {
removedFilesCount++;
}
else {
String baseMessage = LOG_PREFIX + " failed to delete non-wool file that is not present in cache";
String message = baseMessage + FILE_SEPARATOR + file.getName();
log.debug(message);
}
}
}
catch (IllegalArgumentException e) { // because the file does not have a . in its path
if (file.delete()) {
removedFilesCount++;
}
else {
String baseMessage = LOG_PREFIX + " failed to delete file with no extension";
String message = baseMessage + FILE_SEPARATOR + file.getName();
log.debug(message);
}
}
}
}
}
log.debug(LOG_PREFIX + " successfully removed " + removedFilesCount + " from " + directory.getName());
}
}

70
src/main/java/com/sheepit/client/config/DirectoryManager.java
View File

@@ -37,6 +37,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.CompletableFuture;
@AllArgsConstructor
public class DirectoryManager {
@@ -62,6 +63,8 @@ public class DirectoryManager {
return copyFileFromSharedToCache(getSharedPathFor(chunk), getCachePathFor(chunk));
}
private static final Log log = Log.getInstance();
private boolean copyFileFromSharedToCache(String source, String target) {
Path existingArchivePath = Paths.get(source);
Path targetArchivePath = Paths.get(target);
@@ -82,12 +85,12 @@ public class DirectoryManager {
| SecurityException // user is not allowed to create hard-links
ignore) {
// Creating hardlinks might not be supported on some filesystems
Log.getInstance().debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
log.debug("Failed to create hardlink, falling back to copying file to " + targetArchivePath);
Files.copy(existingArchivePath, targetArchivePath, StandardCopyOption.REPLACE_EXISTING);
}
}
catch (IOException e) {
Log.getInstance().error("Error while copying " + source + " from shared downloads directory to working dir");
log.error("Error while copying " + source + " from shared downloads directory to working dir");
return false;
}
@@ -107,10 +110,14 @@ public class DirectoryManager {
this.configuration.getSharedDownloadsDirectory().mkdirs();
if (this.configuration.getSharedDownloadsDirectory().exists() == false) {
System.err.println("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath());
log.error("DirectoryManager::createCacheDir Unable to create common directory " + this.configuration.getSharedDownloadsDirectory().getAbsolutePath());
}
}
}
public CompletableFuture<Void> createCacheDirAsync() {
return CompletableFuture.runAsync(this::createCacheDir);
}
/**
* Cleans working directory and also deletes it if the user hasn't specified a cache directory
@@ -128,51 +135,8 @@ public class DirectoryManager {
* Deletes the working and storage directories
*/
public void cleanWorkingDirectory() {
this.cleanDirectory(this.configuration.getWorkingDirectory());
this.cleanDirectory(this.configuration.getWoolCacheDirectory());
}
/**
* Cleans a directory and removes files in it from the md5 cache
* @param dir representing the directory to be cleaned
* @return false if the dir null, true otherwise
*/
private boolean cleanDirectory(File dir) {
if (dir == null) {
return false;
}
File[] files = dir.listFiles();
if (files != null) {
for (File file : files) {
if (file.isDirectory()) {
Utils.delete(file);
}
else {
try {
String extension = FilenameUtils.getExtension(file.getName()).toLowerCase();
String name = FilenameUtils.removeExtension(file.getName());
if ("wool".equals(extension)) {
// check if the md5 of the file is ok
String md5_local = Utils.md5(file.getAbsolutePath());
if (md5_local.equals(name) == false) {
file.delete();
}
// TODO: remove old one
}
else {
file.delete();
}
}
catch (IllegalArgumentException e) { // because the file does not have an . in his path
file.delete();
}
}
}
}
return true;
new DirectoryCleaner(this.configuration.getWorkingDirectory()).run();
new DirectoryCleaner(this.configuration.getWoolCacheDirectory()).run();
}
/**
@@ -180,7 +144,7 @@ public class DirectoryManager {
* working, storage, and shared downloads directories as long as they are not null
*/
public List<File> getLocalCacheFiles() {
List<File> files_local = new LinkedList<File>();
List<File> filesLocal = new LinkedList<File>();
List<File> files = new LinkedList<File>();
if (this.configuration.getWorkingDirectory() != null) {
File[] filesInDirectory = this.configuration.getWorkingDirectory().listFiles();
@@ -208,10 +172,10 @@ public class DirectoryManager {
String name = FilenameUtils.removeExtension(file.getName());
if ("wool".equals(extension)) {
// check if the md5 of the file is ok
String md5_local = Utils.md5(file.getAbsolutePath());
String md5Local = Utils.md5(file.getAbsolutePath());
if (md5_local.equals(name)) {
files_local.add(file);
if (md5Local.equals(name)) {
filesLocal.add(file);
}
}
}
@@ -220,7 +184,7 @@ public class DirectoryManager {
}
}
return files_local;
return filesLocal;
}
/**

1
src/main/java/com/sheepit/client/config/SettingsLoader.java
View File

@@ -523,7 +523,6 @@ public class SettingsLoader {
if (config.isUserHasSpecifiedACacheDir() == false && cacheDir != null) {
config.setCacheDir(new File(cacheDir.getValue()));
(new DirectoryManager(config)).createCacheDir();
}
if (config.getUIType() == null && ui != null) {

15
src/main/java/com/sheepit/client/main/Worker.java
View File

@@ -40,6 +40,7 @@ import java.time.temporal.ChronoUnit;
import java.util.Calendar;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -476,9 +477,17 @@ public class Worker {
String configFilePath = config.getConfigFilePath() != null ? config.getConfigFilePath() : OS.getOS().getDefaultConfigFilePath();
config.setLogDirectory(log_dir != null ? log_dir : (new File (configFilePath).getParent()));
(new DirectoryManager(config)).createCacheDir();
Log.setInstance(config);
boolean shouldCreateCache = config.isUserHasSpecifiedACacheDir() == false && cache_dir != null;
CompletableFuture<Void> directoryInit;
if (shouldCreateCache) {
directoryInit = (new DirectoryManager(config)).createCacheDirAsync();
}
else {
directoryInit = CompletableFuture.completedFuture(null);
}
Log.setInstance(config);
Log.getInstance().debug("client version " + Configuration.jarVersion);
// Hostname change will overwrite the existing one (default or read from configuration file) but changes will be lost when the client closes
@@ -524,7 +533,7 @@ public class Worker {
((GuiSwing) gui).setSettingsLoader(settingsLoader);
break;
}
Client cli = new Client(gui, config, server);
Client cli = new Client(gui, config, server, directoryInit);
gui.setClient(cli);
ShutdownHook hook = new ShutdownHook(cli);
hook.attachShutDownHook();

33
src/main/java/com/sheepit/client/utils/Md5.java
View File

@@ -27,15 +27,19 @@ import java.nio.file.Paths;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class Md5 {
private static Map<String, String> cache = new HashMap<>();
final class Md5 {
// Maps the file path to a MD5 hash of the file
static final Map<String, String> cache = new ConcurrentHashMap<>();
// TODO: to avoid memory increase, check if files are deleted
public String get(String path) {
private Md5() {}
static synchronized String get(String path) {
String key = getUniqueKey(path);
if (cache.containsKey(key) == false) {
generate(path);
@@ -43,19 +47,20 @@ public class Md5 {
return cache.get(key);
}
private void generate(String path) {
static synchronized void generate(String path) {
String key = getUniqueKey(path);
try {
MessageDigest md = MessageDigest.getInstance("MD5");
InputStream is = Files.newInputStream(Paths.get(path));
DigestInputStream dis = new DigestInputStream(is, md);
byte[] buffer = new byte[8192];
while (dis.read(buffer) > 0) {
// process the entire file
String data;
try (InputStream is = Files.newInputStream(Paths.get(path));
DigestInputStream dis = new DigestInputStream(is, md)) {
byte[] buffer = new byte[256 * 1024]; // 256 KiB buffer
while (dis.read(buffer) > 0) {
// process the entire file
}
data = Utils.convertBinaryToHex(md.digest());
}
String data = Utils.convertBinaryToHex(md.digest());
dis.close();
is.close();
cache.put(key, data);
}
catch (NoSuchAlgorithmException | IOException e) {
@@ -63,7 +68,7 @@ public class Md5 {
}
}
private String getUniqueKey(String path) {
static synchronized String getUniqueKey(String path) {
File file = new File(path);
return Long.toString(file.lastModified()) + '_' + path;
}

37
src/main/java/com/sheepit/client/utils/Utils.java
View File

@@ -22,16 +22,12 @@ package com.sheepit.client.utils;
import com.sheepit.client.zip.ChunkInputStream;
import com.sheepit.client.logger.Log;
import com.sheepit.client.zip.UnzipUtils;
import net.lingala.zip4j.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -63,36 +59,6 @@ public class Utils {
mimeTypes.put(".exr", "image/x-exr");
}
/**
* Extracts (and optionally decrypts) contents of a given zip file to a target directory
*
* @param zipFileName_ Path to the zipfiles
* @param destinationDirectory Path to the target directory where files will be extracted to
* @param password Optional password for decrypting the zip archive which will only be used if it's not null and if the zip is truly encrypted
* @param log The SheepIt Debug log where log messages might be logged into
* @return A status code as an integer, -1 if it encounters a ZipException, 0 otherwise
*/
public static int unzipFileIntoDirectory(String zipFileName_, String destinationDirectory, char[] password, Log log) {
try {
ZipFile zipFile = new ZipFile(zipFileName_);
// unzipParameters.setIgnoreDateTimeAttributes(true);
if (password != null && zipFile.isEncrypted()) {
zipFile.setPassword(password);
}
// zipFile.extractAll(destinationDirectory, unzipParameters);
zipFile.extractAll(destinationDirectory);
}
catch (ZipException e) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
log.debug("Utils::unzipFileIntoDirectory(" + zipFileName_ + "," + destinationDirectory + ") exception " + e + " stacktrace: " + sw.toString());
return -1;
}
return 0;
}
/**
* Takes the list of zip file chunks and combines them to a zip archive in memory to
* extract (and optionally decrypt) the contents to the target directory
@@ -125,8 +91,7 @@ public class Utils {
* @return the string The MD5 checksum
*/
public static String md5(String path_of_file_) {
Md5 md5 = new Md5();
return md5.get(path_of_file_);
return Md5.get(path_of_file_);
}
/**