From 79052b1aa58b9e6814da1814429988a0b40bcb56 Mon Sep 17 00:00:00 2001 From: DaCool <8727384-DaCool@users.noreply.gitlab.com> Date: Sun, 12 Nov 2023 13:36:20 +0000 Subject: [PATCH] Implement MD5 cache clear routine --- .../com/sheepit/client/Configuration.java | 3 +- src/main/java/com/sheepit/client/Md5.java | 115 ++++++++++++++++-- src/main/java/com/sheepit/client/Utils.java | 1 + 3 files changed, 108 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/sheepit/client/Configuration.java b/src/main/java/com/sheepit/client/Configuration.java index 67cb389..e15332f 100644 --- a/src/main/java/com/sheepit/client/Configuration.java +++ b/src/main/java/com/sheepit/client/Configuration.java @@ -265,8 +265,7 @@ import lombok.Data; if (md5_local.equals(name) == false) { file.delete(); } - - // TODO: remove old one + Md5.remove(file.getAbsolutePath()); } else { file.delete(); diff --git a/src/main/java/com/sheepit/client/Md5.java b/src/main/java/com/sheepit/client/Md5.java index ee731da..de5001f 100644 --- a/src/main/java/com/sheepit/client/Md5.java +++ b/src/main/java/com/sheepit/client/Md5.java @@ -28,23 +28,73 @@ import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import java.util.Objects; +/** + * Provides facilities to get an MD5 checksum and cache the results thereof + */ public class Md5 { - private static Map cache = new HashMap<>(); + /** + * Hashmap of all files uniqueMD5keys (last modification time + file) + * and their checksums that have been calculated so far + */ + private static final Map cache = new HashMap<>(); + private static final int cacheCleanTrigger = 64; //After how many cache hits to check and clean the cache + private static int cacheHits = 0; - // TODO: to avoid memory increase, check if files are deleted + + /** + * Removes a file from cache + * @param path Path to the file + */ + public static void remove(String path) { + UniqueMD5Key key = new UniqueMD5Key(path); + cache.remove(key); //Doesn't handle entries that the files moved or had their lastModified changed + } + + /** + * Gets a MD5 checksum either from the cache or computes it and puts it into the cache + * + * @param path Path of the file to get the MD5 checksum for + * @return the string The MD5 checksum + */ public String get(String path) { - String key = getUniqueKey(path); + UniqueMD5Key key = new UniqueMD5Key(path); if (cache.containsKey(key) == false) { generate(path); } + if (cacheHits >= cacheCleanTrigger){ + cacheCheck(); + cacheHits = 0; + } else { + cacheHits++; + } return cache.get(key); } + /** + * Checks cache for files that don't exist anymore according to cached values and removes them + * Catches entries tha fall through the cracks + */ + private void cacheCheck(){ + Iterator> it = cache.entrySet().iterator(); + while (it.hasNext()) { + UniqueMD5Key itKey = (UniqueMD5Key)it.next(); + if (itKey.exists() == false) { + it.remove(); //Iterator instead of foreach because we remove items + } + } + } + + /** + * Calculates the md5 checksum for a given file + * @param path Path to the file to calculate the checksum for + */ private void generate(String path) { - String key = getUniqueKey(path); + UniqueMD5Key key = new UniqueMD5Key(path); try { MessageDigest md = MessageDigest.getInstance("MD5"); InputStream is = Files.newInputStream(Paths.get(path)); @@ -52,18 +102,65 @@ public class Md5 { byte[] buffer = new byte[8192]; while (dis.read(buffer) > 0) ; // process the entire file - String data = Utils.convertBinaryToHex(md.digest()); + String checksum = Utils.convertBinaryToHex(md.digest()); dis.close(); is.close(); - cache.put(key, data); + cache.put(key, checksum); } catch (NoSuchAlgorithmException | IOException e) { cache.put(key, ""); } } +} + +/** + * Used as key for the hashmap that is the cache + * while also making it accessible to check if key is still valid + */ +class UniqueMD5Key { + private final long lastModified; + private final File file; - private String getUniqueKey(String path) { - File file = new File(path); - return Long.toString(file.lastModified()) + '_' + path; + /** + * Returns a UniqueMD5Key for a given filepath + * Automatically fills in lastModified time + * @param filepath The filepath to the file + */ + public UniqueMD5Key(String filepath) { + this.file = new File(filepath); + this.lastModified = this.file.lastModified(); + } + /** + * @see File#exists() + * also takes lastModified into account. + */ + public boolean exists(){ + return file.exists() && lastModified == this.file.lastModified(); + } + + /** + * @see Object#toString() + */ + @Override public String toString() { + return "UniqueMD5Key{" + "lastModified=" + lastModified + ", file=" + file + '}'; + } + + /** + * @see Object#equals(Object) + */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + UniqueMD5Key that = (UniqueMD5Key) o; + return lastModified == that.lastModified && Objects.equals(file, that.file); + } + + /** + * @see Object#hashCode() + */ + @Override public int hashCode() { + return Objects.hash(lastModified, file); } } diff --git a/src/main/java/com/sheepit/client/Utils.java b/src/main/java/com/sheepit/client/Utils.java index da96803..7e83e1a 100644 --- a/src/main/java/com/sheepit/client/Utils.java +++ b/src/main/java/com/sheepit/client/Utils.java @@ -237,6 +237,7 @@ public class Utils { } } file.delete(); + Md5.remove(file.getAbsolutePath()); } /**