Java Code Examples for org.datavec.api.io.labels.PathLabelGenerator

The following examples show how to use org.datavec.api.io.labels.PathLabelGenerator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: DataVec   Source File: LFWLoader.java    License: Apache License 2.0 6 votes vote down vote up
public void load(int batchSize, int numExamples, int numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 
Example 2
Source Project: deeplearning4j   Source File: LFWLoader.java    License: Apache License 2.0 6 votes vote down vote up
public void load(long batchSize, long numExamples, long numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 
Example 3
Source Project: DataVec   Source File: BaseImageRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
protected BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
}
 
Example 4
Source Project: DataVec   Source File: LFWLoader.java    License: Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, int numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        e.printStackTrace();
    }
    return recordReader;
}
 
Example 5
Source Project: DataVec   Source File: JacksonRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 
Example 6
Source Project: deeplearning4j   Source File: BaseImageRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
protected BaseImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
    this.nchw_channels_first = nchw_channels_first;
}
 
Example 7
Source Project: deeplearning4j   Source File: LFWLoader.java    License: Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 
Example 8
Source Project: deeplearning4j   Source File: LFWLoader.java    License: Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 
Example 9
Source Project: deeplearning4j   Source File: JacksonRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 
Example 10
Source Project: DataVec   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    super(height, width, channels, labelGenerator);
}
 
Example 11
Source Project: DataVec   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending no labels. */
public ImageRecordReader(int height, int width, int channels) {
    super(height, width, channels, (PathLabelGenerator) null);
}
 
Example 12
Source Project: DataVec   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                ImageTransform imageTransform) {
    super(height, width, channels, labelGenerator, imageTransform);
}
 
Example 13
Source Project: DataVec   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given  height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, PathLabelGenerator labelGenerator) {
    super(height, width, 1, labelGenerator);
}
 
Example 14
Source Project: DataVec   Source File: BaseImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}
 
Example 15
Source Project: DataVec   Source File: BaseImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                             ImageTransform imageTransform) {
    this(height, width, channels, labelGenerator, null, imageTransform);
}
 
Example 16
Source Project: DataVec   Source File: LFWLoader.java    License: Apache License 2.0 4 votes vote down vote up
public RecordReader getRecordReader(int batchSize, int numExamples, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, new int[] {height, width, channels},
                    useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng);
}
 
Example 17
Source Project: DataVec   Source File: LFWLoader.java    License: Apache License 2.0 4 votes vote down vote up
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
                    train, splitTrainTest, rng);
}
 
Example 18
Source Project: DataVec   Source File: TestImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testImageRecordReaderRegression() throws Exception {

    PathLabelGenerator regressionLabelGen = new TestRegressionLabelGen();

    ImageRecordReader rr = new ImageRecordReader(28, 28, 3, regressionLabelGen);

    File rootDir = new ClassPathResource("/testimages/").getFile();
    FileSplit fs = new FileSplit(rootDir);
    rr.initialize(fs);
    URI[] arr = fs.locations();

    assertTrue(rr.getLabels() == null || rr.getLabels().isEmpty());

    List<Writable> expLabels = new ArrayList<>();
    for(URI u : arr){
        String path = u.getPath();
        expLabels.add(testLabel(path.substring(path.length()-5, path.length())));
    }

    int count = 0;
    while(rr.hasNext()){
        List<Writable> l = rr.next();

        assertEquals(2, l.size());
        assertEquals(expLabels.get(count), l.get(1));

        count++;
    }
    assertEquals(6, count);

    //Test batch ops:
    rr.reset();

    List<List<Writable>> b1 = rr.next(3);
    List<List<Writable>> b2 = rr.next(3);
    assertFalse(rr.hasNext());

    NDArrayRecordBatch b1a = (NDArrayRecordBatch)b1;
    NDArrayRecordBatch b2a = (NDArrayRecordBatch)b2;
    assertEquals(2, b1a.getArrays().size());
    assertEquals(2, b2a.getArrays().size());

    NDArrayWritable l1 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(0).toDouble(),
            expLabels.get(1).toDouble(), expLabels.get(2).toDouble()}, new int[]{3,1}));
    NDArrayWritable l2 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(3).toDouble(),
            expLabels.get(4).toDouble(), expLabels.get(5).toDouble()}, new int[]{3,1}));

    INDArray act1 = b1a.getArrays().get(1);
    INDArray act2 = b2a.getArrays().get(1);
    assertEquals(l1.get(), act1);
    assertEquals(l2.get(), act2);
}
 
Example 19
Source Project: DataVec   Source File: JacksonRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator) {
    this(selection, mapper, shuffle, rngSeed, labelGenerator, -1);
}
 
Example 20
Source Project: DataVec   Source File: BalancedPathFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, 0)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator) {
    this(random, extensions, labelGenerator, 0, 0, 0, 0);
}
 
Example 21
Source Project: DataVec   Source File: BalancedPathFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, int maxPathsPerLabel) {
    this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 
Example 22
Source Project: DataVec   Source File: BalancedPathFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator,
                int maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 
Example 23
Source Project: DataVec   Source File: BalancedPathFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, int maxPaths, int maxLabels,
                int maxPathsPerLabel) {
    this(random, null, labelGenerator, maxPaths, maxLabels, 0, maxPathsPerLabel);
}
 
Example 24
Source Project: DataVec   Source File: BalancedPathFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator, int maxLabels,
                int maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel);
}
 
Example 25
Source Project: deeplearning4j   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator.
 * Output format is NCHW (channels first) - [numExamples, channels, height, width]
 */
public ImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
    super(height, width, channels, labelGenerator);
}
 
Example 26
Source Project: deeplearning4j   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending no labels - in NCHW (channels first) format */
public ImageRecordReader(long height, long width, long channels) {
    super(height, width, channels, (PathLabelGenerator) null);
}
 
Example 27
Source Project: deeplearning4j   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator.
 * Output format is NCHW (channels first) - [numExamples, channels, height, width] */
public ImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
                ImageTransform imageTransform) {
    super(height, width, channels, labelGenerator, imageTransform);
}
 
Example 28
Source Project: deeplearning4j   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator.<br>
 * If {@code nchw_channels_first == true} output format is NCHW (channels first) - [numExamples, channels, height, width]<br>
 * If {@code nchw_channels_first == false} output format is NHWC (channels last) - [numExamples, height, width, channels]<br>
 */
public ImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                         ImageTransform imageTransform) {
    super(height, width, channels, nchw_channels_first, labelGenerator, null, imageTransform);
}
 
Example 29
Source Project: deeplearning4j   Source File: ImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
/** Loads images with given  height, width, and channels, appending labels returned by the generator
 * Output format is NCHW (channels first) - [numExamples, channels, height, width]*/
public ImageRecordReader(long height, long width, PathLabelGenerator labelGenerator) {
    super(height, width, 1, labelGenerator);
}
 
Example 30
Source Project: deeplearning4j   Source File: BaseImageRecordReader.java    License: Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}