Raincloud/sheepit-shadow-nabber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract zips while reading the chunks, instead reading them to memory first

Browse Source
This commit is contained in:
M*C*O
2023-11-11 08:50:11 +00:00
committed by Sheepit Renderfarm
parent 154108dc38
commit 9f825b8d0f
2 changed files with 57 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/com/sheepit/client/Utils.java
View File

@@ -45,6 +45,7 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Provides various general utility methods for the SheepIt client codebase
@@ -105,18 +106,11 @@ public class Utils {
*/
public static int unzipChunksIntoDirectory(List<String> full_path_chunks, String destinationDirectory, char[] password, Log log) {
try {
// STEP 1: assemble the chunks into an actual zip (in RAM)
ByteArrayOutputStream unzippedData = new ByteArrayOutputStream();
for (String full_path_chunk: full_path_chunks) {
byte[] data = Files.readAllBytes(Paths.get(full_path_chunk));
unzippedData.write(data);
}
byte[] full_data = unzippedData.toByteArray();
// STEP 1: Create a ChunkInputStream, which will read the chunks' contents in order
ChunkInputStream chunkInputStream = new ChunkInputStream(full_path_chunks.stream().map(Paths::get).collect(Collectors.toList()));
// STEP 2: unzip the zip like before
ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(full_data));
ZipInputStream zipInputStream = new ZipInputStream(chunkInputStream);
if (password != null) {
zipInputStream.setPassword(password);
}