Java Code Examples for org.apache.commons.io.FileUtils#checksum()

@VisibleForTesting
@SneakyThrows(IOException.class)
void checksumFile(File file, Checksum checksum) {
    Path path = file.toPath();
    checksum.update(MountableFile.getUnixFileMode(path));
    if (file.isDirectory()) {
        try (Stream<Path> stream = Files.walk(path)) {
            stream.filter(it -> it != path).forEach(it -> {
                checksumFile(it.toFile(), checksum);
            });
        }
    } else {
        FileUtils.checksum(file, checksum);
    }
}
 

@Override
public ComputationGraph initPretrained(PretrainedType pretrainedType) throws IOException {
    String remoteUrl = pretrainedUrl(pretrainedType);
    if (remoteUrl == null)
        throw new UnsupportedOperationException(
                "Pretrained " + pretrainedType + " weights are not available for this model.");

    // Set up file locations
    String localFilename = modelPrettyName() + ".zip";

    File rootCacheDir = DL4JResources.getDirectory(ResourceType.ZOO_MODEL, modelFamily());
    File cachedFile = new File(rootCacheDir, localFilename);

    // Download the file if necessary
    if (!cachedFile.exists()) {
        log.info("Downloading model to " + cachedFile.toString());
        FileUtils.copyURLToFile(new URL(remoteUrl), cachedFile);
    } else {
        log.info("Using cached model at " + cachedFile.toString());
    }

    // Validate the checksum - ensure this is the correct file
    long expectedChecksum = pretrainedChecksum(pretrainedType);
    if (expectedChecksum != 0L) {
        log.info("Verifying download...");
        Checksum adler = new Adler32();
        FileUtils.checksum(cachedFile, adler);
        long localChecksum = adler.getValue();
        log.info("Checksum local is " + localChecksum + ", expecting " + expectedChecksum);

        if (expectedChecksum != localChecksum) {
            log.error("Checksums do not match. Cleaning up files and failing...");
            cachedFile.delete();
            throw new IllegalStateException(
                    String.format("Pretrained model file for model %s failed checksum.", this.modelPrettyName()));
        }
    }

    // Load the .zip file to a ComputationGraph
    try {
        return ModelSerializer.restoreComputationGraph(cachedFile);
    } catch (Exception ex) {
        System.err.println("Failed to load model");
        ex.printStackTrace();
        return null;
    }
}
 

@Override
public ComputationGraph initPretrained(PretrainedType pretrainedType) throws IOException {
    String remoteUrl = pretrainedUrl(pretrainedType);
    if (remoteUrl == null)
        throw new UnsupportedOperationException(
                "Pretrained " + pretrainedType + " weights are not available for this model.");

    // Set up file locations
    String localFilename = modelPrettyName() + ".zip";

    File rootCacheDir = DL4JResources.getDirectory(ResourceType.ZOO_MODEL, modelFamily());
    File cachedFile = new File(rootCacheDir, localFilename);

    // Download the file if necessary
    if (!cachedFile.exists()) {
        log.info("Downloading model to " + cachedFile.toString());
        FileUtils.copyURLToFile(new URL(remoteUrl), cachedFile);
    } else {
        log.info("Using cached model at " + cachedFile.toString());
    }

    // Validate the checksum - ensure this is the correct file
    long expectedChecksum = pretrainedChecksum(pretrainedType);
    if (expectedChecksum != 0L) {
        log.info("Verifying download...");
        Checksum adler = new Adler32();
        FileUtils.checksum(cachedFile, adler);
        long localChecksum = adler.getValue();
        log.info("Checksum local is " + localChecksum + ", expecting " + expectedChecksum);

        if (expectedChecksum != localChecksum) {
            log.error("Checksums do not match. Cleaning up files and failing...");
            cachedFile.delete();
            throw new IllegalStateException(
                    String.format("Pretrained model file for model %s failed checksum.", this.modelPrettyName()));
        }
    }

    // Load the .zip file to a ComputationGraph
    try {
        return ModelSerializer.restoreComputationGraph(cachedFile);
    } catch (Exception ex) {
        System.err.println("Failed to load model");
        ex.printStackTrace();
        return null;
    }
}
 

/**
 * Downloads and extracts the local dataset.
 *
 * @throws IOException
 */
public void downloadAndExtract(DataSetType set) throws IOException {
    String localFilename = new File(remoteDataUrl(set)).getName();
    File tmpFile = new File(System.getProperty("java.io.tmpdir"), localFilename);
    File localCacheDir = getLocalCacheDir();

    // check empty cache
    if(localCacheDir.exists()) {
        File[] list = localCacheDir.listFiles();
        if(list == null || list.length == 0)
            localCacheDir.delete();
    }

    File localDestinationDir = new File(localCacheDir, dataSetName(set));
    if(!localDestinationDir.exists()) {
        localCacheDir.mkdirs();
        tmpFile.delete();
        log.info("Downloading dataset to " + tmpFile.getAbsolutePath());
        FileUtils.copyURLToFile(new URL(remoteDataUrl(set)), tmpFile);
    } else {
        //Directory exists and is non-empty - assume OK
        log.info("Using cached dataset at " + localCacheDir.getAbsolutePath());
        return;
    }

    if(expectedChecksum(set) != 0L) {
        log.info("Verifying download...");
        Checksum adler = new Adler32();
        FileUtils.checksum(tmpFile, adler);
        long localChecksum = adler.getValue();
        log.info("Checksum local is " + localChecksum + ", expecting "+expectedChecksum(set));

        if(expectedChecksum(set) != localChecksum) {
            log.error("Checksums do not match. Cleaning up files and failing...");
            tmpFile.delete();
            throw new IllegalStateException( "Dataset file failed checksum: " + tmpFile + " - expected checksum " + expectedChecksum(set)
            + " vs. actual checksum " + localChecksum + ". If this error persists, please open an issue at https://github.com/deeplearning4j/deeplearning4j.");
        }
    }

    try {
        ArchiveUtils.unzipFileTo(tmpFile.getAbsolutePath(), localCacheDir.getAbsolutePath(), false);
    } catch (Throwable t){
        //Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state
        if(localCacheDir.exists())
            FileUtils.deleteDirectory(localCacheDir);
        throw t;
    }
}
 

/**
 * Returns a pretrained model for the given dataset, if available.
 *
 * @param pretrainedType
 * @return
 * @throws IOException
 */
public <M extends Model> M initPretrained(PretrainedType pretrainedType) throws IOException {
    String remoteUrl = pretrainedUrl(pretrainedType);
    if (remoteUrl == null)
        throw new UnsupportedOperationException(
                        "Pretrained " + pretrainedType + " weights are not available for this model.");

    String localFilename = new File(remoteUrl).getName();

    File rootCacheDir = DL4JResources.getDirectory(ResourceType.ZOO_MODEL, modelName());
    File cachedFile = new File(rootCacheDir, localFilename);

    if (!cachedFile.exists()) {
        log.info("Downloading model to " + cachedFile.toString());
        FileUtils.copyURLToFile(new URL(remoteUrl), cachedFile);
    } else {
        log.info("Using cached model at " + cachedFile.toString());
    }

    long expectedChecksum = pretrainedChecksum(pretrainedType);
    if (expectedChecksum != 0L) {
        log.info("Verifying download...");
        Checksum adler = new Adler32();
        FileUtils.checksum(cachedFile, adler);
        long localChecksum = adler.getValue();
        log.info("Checksum local is " + localChecksum + ", expecting " + expectedChecksum);

        if (expectedChecksum != localChecksum) {
            log.error("Checksums do not match. Cleaning up files and failing...");
            cachedFile.delete();
            throw new IllegalStateException(
                            "Pretrained model file failed checksum. If this error persists, please open an issue at https://github.com/deeplearning4j/deeplearning4j.");
        }
    }

    if (modelType() == MultiLayerNetwork.class) {
        return (M) ModelSerializer.restoreMultiLayerNetwork(cachedFile);
    } else if (modelType() == ComputationGraph.class) {
        return (M) ModelSerializer.restoreComputationGraph(cachedFile);
    } else {
        throw new UnsupportedOperationException(
                        "Pretrained models are only supported for MultiLayerNetwork and ComputationGraph.");
    }
}