path: root/src/java/com/jogamp/common/util/SHASum.java

/**
 * Copyright 2019 JogAmp Community. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are
 * permitted provided that the following conditions are met:
 *
 *    1. Redistributions of source code must retain the above copyright notice, this list of
 *       conditions and the following disclaimer.
 *
 *    2. Redistributions in binary form must reproduce the above copyright notice, this list
 *       of conditions and the following disclaimer in the documentation and/or other materials
 *       provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY JogAmp Community ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JogAmp Community OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are those of the
 * authors and should not be interpreted as representing official policies, either expressed
 * or implied, of JogAmp Community.
 */

package com.jogamp.common.util;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;

import com.jogamp.common.GlueGenVersion;
import com.jogamp.common.util.cache.TempFileCache;
import com.jogamp.common.util.cache.TempJarCache;

import jogamp.common.Debug;

/**
 * Utility class to produce secure hash (SHA) sums over diverse input sources.
 * <p>
 * See {@link #updateDigest(MessageDigest, List)}
 * </p>
 * <p>
 * This implementation is being utilized at JogAmp build time to produce various
 * SHA256 sums over sources, class files and native libraries to ensure their identity.
 * See {@link JogampVersion#getImplementationSHA256Sources()},
 * {@link JogampVersion#getImplementationSHA256Classes()}
 * and {@link JogampVersion#getImplementationSHA256Natives()}.
 * </p>
 * <p>
 * {@link JogampVersion#getImplementationSHA256Sources()} for module gluegen is produced via:
 * <pre>
 * java -cp build/gluegen-rt.jar com.jogamp.common.util.SHASum --algorithm 256 --exclude ".*\\.log" --exclude "make/lib/toolchain" src jcpp/src make
 * </pre>
 * </p>
 * @see #SHASum(MessageDigest, List, List, List)
 * @see #compute(boolean)
 * @see TempJarSHASum
 * @see #main(String[])
 */
public class SHASum {
    private static final boolean DEBUG = Debug.debug("SHASum");

    /**
     * {@link MessageDigest#update(byte[], int, int) Updates} the given {@code digest}
     * with the bytes contained by the files denoted by the given {@code filenames} in the given order.
     * <p>
     * To retrieve the list of all files traversing through directories, one may use {@link IOUtil#filesOf(List, List, List)}.
     * </p>
     * <p>
     * The SHA implementation is sensitive to the order of input bytes and hence the given filename order.
     * </p>
     * <p>
     * It is advised to pass given list of filenames in lexicographically sorted order to ensure reproducible outcome across all platforms,
     * one may use {@link #sort(ArrayList)}.
     * </p>
     * <p>
     * As an example, one could write
     * <pre>
     * final MessageDigest digest = ...;
     * final long totalBytes = updateDigest(digest, sort(IOUtil.filesOf(Arrays.asList("sources"), null, null)));
     * </pre>
     * </p>
     * @param digest to be updated digest
     * @param filenames list of filenames denoting files, which bytes will be used to update the digest
     * @return total number of bytes read.
     * @throws FileNotFoundException see {@link FileInputStream#FileInputStream(String)}
     * @throws IOException see {@link InputStream#read(byte[])}
     */
    public static long updateDigest(final MessageDigest digest, final List<String> filenames) throws IOException {
        long numBytes = 0;
        final byte buffer[] = new byte[4096]; // avoid Platform.getMachineDataInfo().pageSizeInBytes() due to native dependency
        for(int i=0; i<filenames.size(); i++) {
            final InputStream in = new BufferedInputStream(new FileInputStream(filenames.get(i)));
            try {
                while (true) {
                    int count;
                    if ((count = in.read(buffer)) == -1) {
                        break;
                    }
                    digest.update(buffer, 0, count);
                    numBytes += count;
                }
            } finally {
                in.close();
            }
        }
        return numBytes;
    }

    /**
     * Simple helper to print the given byte-array into a string, here appended to StringBuilder
     * @param shasum the given byte-array
     * @param sb optional pre-existing StringBuilder, may be null
     * @return return given or new StringBuilder with appended hex-string
     */
    public static StringBuilder toHexString(final byte[] shasum, StringBuilder sb) {
        if( null == sb ) {
            sb = new StringBuilder();
        }
        for(int i=0; i<shasum.length; i++) {
            sb.append(String.format((Locale)null, "%02x", shasum[i]));
        }
        return sb;
    }

    /**
     * Returns the sorted list of given strings using {@link String#compareTo(String)}'s lexicographically comparison.
     * @param source given input strings
     * @return sorted list of given strings
     */
    public static List<String> sort(final ArrayList<String> source) {
        final String s[] = source.toArray(new String[source.size()]);
        Arrays.sort(s, 0, s.length, null);
        return Arrays.asList(s);
    }

    final MessageDigest digest;
    final List<String> origins;
    final List<Pattern> excludes, includes;

    /**
     * Instance to ensure proper {@link #compute(boolean)} of identical SHA sums over same contents within given paths across machines.
     * <p>
     * Instantiation of this class is lightweight, {@link #compute(boolean)} performs all operations.
     * </p>
     *
     * @param digest the SHA algorithm
     * @param origins the mandatory path origins to be used for {@link IOUtil#filesOf(List, List, List)}
     * @param excludes the optional exclude patterns to be used for {@link IOUtil#filesOf(List, List, List)}
     * @param includes the optional include patterns to be used for {@link IOUtil#filesOf(List, List, List)}
     * @throws IllegalArgumentException
     * @throws IOException
     * @throws URISyntaxException
     */
    public SHASum(final MessageDigest digest, final List<String> origins, final List<Pattern> excludes, final List<Pattern> includes) {
        this.digest = digest;
        this.origins = origins;
        this.excludes = excludes;
        this.includes = includes;
    }

    /**
     * Implementation gathers all files traversing through given paths via {@link IOUtil#filesOf(List, List, List)},
     * sorts the resulting file list via {@link #sort(ArrayList)} and finally
     * calculates the SHA sum over its byte content via {@link #updateDigest(MessageDigest, List)}.
     * <p>
     * This ensures identical SHA sums over same contents within given paths across machines.
     * </p>
     * <p>
     * This method is heavyweight and performs all operations.
     * </p>
     *
     * @param verbose if true, all used files will be dumped as well as the digest result
     * @return the resulting SHA value
     * @throws IOException
     */
    public final byte[] compute(final boolean verbose) throws IOException {
        final List<String> fnamesS = SHASum.sort(IOUtil.filesOf(origins, excludes, includes));
        if( verbose ) {
            for(int i=0; i<fnamesS.size(); i++) {
                System.err.println(fnamesS.get(i));
            }
        }
        final long numBytes = SHASum.updateDigest(digest, fnamesS);
        final byte[] shasum = digest.digest();
        if( verbose ) {
            System.err.println("Digested "+numBytes+" bytes, shasum size "+shasum.length+" bytes");
            System.err.println("Digested result: "+SHASum.toHexString(shasum, null).toString());
        }
        return shasum;
    }

    public final List<String> getOrigins() { return origins; }
    public final List<Pattern> getExcludes() { return excludes; }
    public final List<Pattern> getIncludes() { return includes; }

    /**
     * {@link SHASum} specialization utilizing {@link TempJarCache} to access jar file content for SHA computation
     */
    public static class TempJarSHASum extends SHASum {
        /**
         * Instance to ensure proper {@link #compute(boolean)} of identical SHA sums over same contents within given paths across machines.
         * <p>
         * Instantiation of this class is lightweight, {@link #compute(boolean)} performs all operations.
         * </p>
         * <p>
         * {@link TempJarCache#getTempFileCache()}'s {@link TempFileCache#getTempDir()} is used as origin for {@link IOUtil#filesOf(List, List, List)}
         * </p>
         *
         * @param digest the SHA algorithm
         * @param jarclazz a class from the desired classpath jar file used for {@link TempJarCache#addAll(Class, com.jogamp.common.net.Uri)}
         * @param excludes the optional exclude patterns to be used for {@link IOUtil#filesOf(List, List, List)}
         * @param includes the optional include patterns to be used for {@link IOUtil#filesOf(List, List, List)}
         * @throws SecurityException
         * @throws IllegalArgumentException
         * @throws IOException
         * @throws URISyntaxException
         */
        public TempJarSHASum(final MessageDigest digest, final Class<?> jarclazz, final List<Pattern> excludes, final List<Pattern> includes)
                throws SecurityException, IllegalArgumentException, IOException, URISyntaxException
        {
            super(digest, Arrays.asList(IOUtil.slashify(TempJarCache.getTempFileCache().getTempDir().getAbsolutePath(), false, false)),
                  excludes, includes);
            TempJarCache.addAll(jarclazz, JarUtil.getJarFileUri(jarclazz.getName(), jarclazz.getClassLoader()));
        }

        public final String getOrigin() { return origins.get(0); }
    }

    /**
     * Main entry point taking var-arg path or gnu-arguments with a leading '--'.
     * <p>
     * Implementation gathers all files traversing through given paths via {@link IOUtil#filesOf(List, List, List)},
     * sorts the resulting file list via {@link #sort(ArrayList)} and finally
     * calculates the SHA sum over its byte content via {@link #updateDigest(MessageDigest, List)}.
     * This ensures identical SHA sums over same contents within given paths.
     * </p>
     * <p>
     * Example to calculate the SHA-256 over our source files as performed for {@link JogampVersion#getImplementationSHA256Sources()}
     * <pre>
     * java -cp build/gluegen-rt.jar com.jogamp.common.util.SHASum --algorithm 256 --exclude ".*\\.log" --exclude "make/lib/toolchain" src jcpp/src make
     * </pre>
     * </p>
     * <p>
     * To validate the implementation, one can gather the sorted list of files (to ensure same order)
     * <pre>
     * java -cp build/gluegen-rt.jar com.jogamp.common.util.SHASum --listfilesonly --exclude ".*\\.log" --exclude "make/lib/toolchain" src jcpp/src make >& java.sorted.txt
     * </pre>
     * and then calculate the shasum independently
     * <pre>
     * find `cat java.sorted.txt` -exec cat {} + | shasum -a 256 -b - | awk '{print $1}'
     * </pre>
     * </p>
     * @param args
     * @throws IOException
     * @throws URISyntaxException
     * @throws IllegalArgumentException
     */
    public static void main(final String[] args) throws IOException {
        boolean listFilesOnly = false;
        int shabits = 256;
        int i;
        final ArrayList<String> pathU = new ArrayList<String>();
        final ArrayList<Pattern> excludes = new ArrayList<Pattern>();
        final ArrayList<Pattern> includes = new ArrayList<Pattern>();
        {
            for(i=0; i<args.length; i++) {
                if(null != args[i]) {
                    if( args[i].startsWith("--") ) {
                        // options
                        if( args[i].equals("--algorithm")) {
                            shabits = Integer.parseInt(args[++i]);
                        } else if( args[i].equals("--exclude")) {
                            excludes.add(Pattern.compile(args[++i]));
                            if( DEBUG ) {
                                System.err.println("adding exclude: <"+args[i]+"> -> <"+excludes.get(excludes.size()-1)+">");
                            }
                        } else if( args[i].equals("--include")) {
                            includes.add(Pattern.compile(args[++i]));
                            if( DEBUG ) {
                                System.err.println("adding include: <"+args[i]+"> -> <"+includes.get(includes.size()-1)+">");
                            }
                        } else if( args[i].equals("--listfilesonly")) {
                            listFilesOnly = true;
                        } else {
                            System.err.println("Abort, unknown argument: "+args[i]);
                            return;
                        }
                    } else {
                        pathU.add(args[i]);
                        if( DEBUG ) {
                            System.err.println("adding path: <"+args[i]+">");
                        }
                    }
                }
            }
            if( listFilesOnly ) {
                final List<String> fnamesS = sort(IOUtil.filesOf(pathU, excludes, includes));
                for(i=0; i<fnamesS.size(); i++) {
                    System.out.println(fnamesS.get(i));
                }
                return;
            }
        }
        final String shaalgo = "SHA-"+shabits;
        final MessageDigest digest;
        try {
            digest = MessageDigest.getInstance(shaalgo);
        } catch (final NoSuchAlgorithmException e) {
            System.err.println("Abort, implementation for "+shaalgo+" not available: "+e.getMessage());
            return;
        }
        final SHASum shaSum = new SHASum(digest, pathU, excludes, includes);
        System.out.println(toHexString(shaSum.compute(DEBUG), null).toString());
    }
}