/* * Copyright (C) 2010-2014 Laurent CLOUET * Author Laurent CLOUET * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; version 2 * of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package com.sheepit.client; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.io.inputstream.ZipInputStream; import net.lingala.zip4j.model.LocalFileHeader; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * Provides various general utility methods for the SheepIt client codebase */ public class Utils { /** * This is a hashmap of media types currently consisting of two image formats, tga and exr * The first string is the filename extension. * The second string is the media/MIME type */ private static Map mimeTypes = new HashMap<>(); static { mimeTypes.put(".tga", "image/tga"); mimeTypes.put(".exr", "image/x-exr"); } /** * Extracts (and optionally decrypts) contents of a given zip file to a target directory * * @param zipFileName_ Path to the zipfiles * @param destinationDirectory Path to the target directory where files will be extracted to * @param password Optional password for decrypting the zip archive which will only be used if it's not null and if the zip is truly encrypted * @param log The SheepIt Debug log where log messages might be logged into * @return A status code as an integer, -1 if it encounters a ZipException, 0 otherwise */ public static int unzipFileIntoDirectory(String zipFileName_, String destinationDirectory, char[] password, Log log) { try { ZipFile zipFile = new ZipFile(zipFileName_); // unzipParameters.setIgnoreDateTimeAttributes(true); if (password != null && zipFile.isEncrypted()) { zipFile.setPassword(password); } // zipFile.extractAll(destinationDirectory, unzipParameters); zipFile.extractAll(destinationDirectory); } catch (ZipException e) { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); log.debug("Utils::unzipFileIntoDirectory(" + zipFileName_ + "," + destinationDirectory + ") exception " + e + " stacktrace: " + sw.toString()); return -1; } return 0; } /** * Takes the list of zip file chunks and combines them to a zip archive in memory to * extract (and optionally decrypt) the contents to the target directory * * @param full_path_chunks List of paths to the chunks of which a zip file will be constructed from * @param destinationDirectory Path to the target directory where files will be extracted to * @param password Optional password for decrypting the zip archive which will only be used if it's not null * @param log The SheepIt Debug log where log messages might be logged into * @return A status code as an integer, -1 if it encounters a IOException, 0 otherwise */ public static int unzipChunksIntoDirectory(List full_path_chunks, String destinationDirectory, char[] password, Log log) { try { // STEP 1: Create a ChunkInputStream, which will read the chunks' contents in order ChunkInputStream chunkInputStream = new ChunkInputStream(full_path_chunks.stream().map(Paths::get).collect(Collectors.toList())); // STEP 2: unzip the zip like before ZipInputStream zipInputStream = new ZipInputStream(chunkInputStream); if (password != null) { zipInputStream.setPassword(password); } LocalFileHeader fileHeader = null; while ((fileHeader = zipInputStream.getNextEntry()) != null) { String outFilePath = destinationDirectory + File.separator + fileHeader.getFileName(); File outFile = new File(outFilePath); //Checks if the file is a directory if (fileHeader.isDirectory()) { outFile.mkdirs(); continue; } File parentDir = outFile.getParentFile(); if (parentDir.exists() == false) { parentDir.mkdirs(); } FileOutputStream os = new FileOutputStream(outFile); int readLen = -1; byte[] buff = new byte[1024]; //Loop until End of File and write the contents to the output stream while ((readLen = zipInputStream.read(buff)) != -1) { os.write(buff, 0, readLen); } os.close(); } zipInputStream.close(); } catch (IOException e) { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); log.debug("Utils::unzipChunksIntoDirectory exception " + e + " stacktrace: " + sw.toString()); return -1; } return 0; } /** * Gets a MD5 checksum either from the cache or computes it and puts it into the cache * * @param path_of_file_ Path of the file to get the MD5 checksum for * @return the string The MD5 checksum */ public static String md5(String path_of_file_) { Md5 md5 = new Md5(); return md5.get(path_of_file_); } /** * Takes an array of bytes and encodes them as a string of hexadecimal digits * @param bytes An array of bytes to be encoded, the order of which dictates the output in big-endian * @return A string of hexadecimal digits, two digits for a byte each. Constructed from left to right. */ public static String convertBinaryToHex(byte[] bytes) { StringBuilder hexStringBuilder = new StringBuilder(); for (byte aByte : bytes) { char[] hex = new char[2]; hex[0] = Character.forDigit((aByte >> 4) & 0xF, 16); hex[1] = Character.forDigit((aByte & 0xF), 16); hexStringBuilder.append(new String(hex)); } return hexStringBuilder.toString(); } /** * Returns the latest (highest) of a modification time of a file in the directory recursively * @param directory_ The root directory from which to search from * @return The latest (highest) modification time in milliseconds since epoch or 0 if no files exist in the target directory tree */ public static double lastModificationTime(File directory_) { double max = 0.0; if (directory_.isDirectory()) { File[] list = directory_.listFiles(); if (list != null) { for (File aFile : list) { double max1 = lastModificationTime(aFile); if (max1 > max) { max = max1; } } } } else if (directory_.isFile()) { return directory_.lastModified(); } return max; } /** * Deletes a file or directory recursively * @param file The filesystem element to be deleted */ public static void delete(File file) { if (file == null) { return; } if (file.isDirectory()) { String[] files = file.list(); if (files != null && files.length != 0) { for (String temp : files) { File fileDelete = new File(file, temp); delete(fileDelete); } } } file.delete(); } /** * Converts a number string to a number. * @param in The number string with an ending up to Tera (10^12) * Example inputs: "32", "10k", "100K", "100G", "1.3G", "0.4T" * @return A numerical representation of the number string */ public static long parseNumber(String in) { in = in.trim(); in = in.replaceAll(",", "."); try { return Long.parseLong(in); } catch (NumberFormatException e) { } final Matcher m = Pattern.compile("([\\d.,]+)\\s*(\\w)").matcher(in); m.find(); long scale = 1; switch (Character.toUpperCase(m.group(2).charAt(0))) { case 'T': scale *= 1000; //Multiply by 1000 each time the higher the metric prefix case 'G': //Note the lack of break statements, thus it executes through scale *= 1000; case 'M': scale *= 1000; case 'K': scale *= 1000; break; } return Math.round(Double.parseDouble(m.group(1)) * scale); } /** * Gives a human-readable representation in hours, minutes and seconds when a given date will be reached * @param date The target time * @return A string in the format of Xh Ymin Ys as long as none of them are zero * Examples: 69h 42min 13s, 20min, 1h 23s */ public static String humanDuration(Date date) { Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT")); calendar.setTime(date); int hours = (calendar.get(Calendar.DAY_OF_MONTH) - 1) * 24 + calendar.get(Calendar.HOUR_OF_DAY); int minutes = calendar.get(Calendar.MINUTE); int seconds = calendar.get(Calendar.SECOND); String output = ""; if (hours > 0) { output += hours + "h "; } if (minutes > 0) { output += minutes + "min "; } if (seconds > 0) { output += seconds + "s"; } return output; } /** * Checks for a given path if there is enough free space i.e. at-least over half a megabyte. * If the first check fails, it polls again for up to 3 times with increasing delay * to work around a Java limitation where getUsableSpace might just return 0 on a busy disk * @param destination_ The path to check the for enough free space for * @param log The SheepIt Debug log where log messages might be logged into * @return True if there is not enough free space, false otherwise */ public static boolean noFreeSpaceOnDisk(String destination_, Log log) { try { File file = new File(destination_); for (int i = 0; i < 5; i++) { //We poll repeatedly because getUsableSpace() might just return 0 on busy disk IO long space = file.getUsableSpace(); if (space > 512 * 1024) { // at least the same amount as Server.HTTPGetFile return false; // If we are not "full", we are done, no need for additional polling } else if (i < 4) { long time = (long) ( Math.random() * (100 - 50 + 1) + 50 + //Wait between 50 and 100 milliseconds, (i * 500) //add 500 ms on each failed poll ); log.debug("Utils::Not enough free disk space(" + space + ") encountered on try " + i + ", waiting " + time + "ms"); Thread.sleep(time); } } return true; } catch (SecurityException | InterruptedException e) { } return false; } /** * Tries to detect media/MIME type of given file based on characteristics like the file contents or file extension * @param file Path to the file for which to detect the MIME type for * @return The MIME type of the file * @throws IOException If an I/O error occurs */ public static String findMimeType(String file) throws IOException { String mimeType = Files.probeContentType(Paths.get(file)); if (mimeType == null) { InputStream stream = new BufferedInputStream(new FileInputStream(file)); mimeType = URLConnection.guessContentTypeFromStream(stream); } if (mimeType == null) { mimeType = URLConnection.guessContentTypeFromName(file); } if (mimeType == null || (mimeType.equals("image/aces") && file.toLowerCase().endsWith(".exr"))) { try { String extension = file.substring(file.lastIndexOf('.')); mimeType = mimeTypes.get(extension); } catch (IndexOutOfBoundsException e) { e.printStackTrace(); } } return mimeType; } /** * Format a number of bytes into a human-readable format with metric prefixes (base 1024) * @param bytes The amount of bytes to convert * @return Human-readable formatted string (respecting locale of the machine) * representing the amount of bytes * Examples: 4.20GB, 20.00TB, 3.69MB * In a different locale (here for German): * Examples: 4,20GB, 20,00TB, 3,69MB */ public static String formatDataConsumption(long bytes) { float divider = 0; String suffix = ""; if (bytes > 1099511627776f) { // 1TB divider = 1099511627776f; suffix = "TB"; } else if (bytes > 1073741824) { // 1GB divider = 1073741824; suffix = "GB"; } else { // 1MB divider = 1048576; suffix = "MB"; } return String.format("%.2f%s", (bytes / divider), suffix); } /** * Sometimes on Windows, delete of file return false for no obvious reason. * Wait a bit and try again */ public static boolean deleteFile(File file) { if (file.delete() == false) { // maybe the system was busy try { Thread.sleep(4000); } catch (InterruptedException e) { } System.gc(); if (file.delete() == false) { file.deleteOnExit(); // nothing more can be done... return false; } } return true; } }