org.datavec.api.io.labels.PathLabelGenerator Java Examples

The following examples show how to use org.datavec.api.io.labels.PathLabelGenerator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public void load(long batchSize, long numExamples, long numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 
Example #2
Source File: LFWLoader.java    From DataVec with Apache License 2.0 6 votes vote down vote up
public void load(int batchSize, int numExamples, int numLabels, PathLabelGenerator labelGenerator,
                double splitTrainTest, Random rng) {
    if (!imageFilesExist()) {
        if (!fullDir.exists() || fullDir.listFiles() == null || fullDir.listFiles().length == 0) {
            fullDir.mkdir();

            if (useSubset) {
                log.info("Downloading {} subset...", localDir);
                downloadAndUntar(lfwSubsetData, fullDir);
            } else {
                log.info("Downloading {}...", localDir);
                downloadAndUntar(lfwData, fullDir);
                downloadAndUntar(lfwLabel, fullDir);
            }
        }
    }
    FileSplit fileSplit = new FileSplit(fullDir, ALLOWED_FORMATS, rng);
    BalancedPathFilter pathFilter = new BalancedPathFilter(rng, ALLOWED_FORMATS, labelGenerator, numExamples,
                    numLabels, 0, batchSize, null);
    inputSplit = fileSplit.sample(pathFilter, numExamples * splitTrainTest, numExamples * (1 - splitTrainTest));
}
 
Example #3
Source File: BaseImageRecordReader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
protected BaseImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
    this.nchw_channels_first = nchw_channels_first;
}
 
Example #4
Source File: BaseImageRecordReader.java    From DataVec with Apache License 2.0 5 votes vote down vote up
protected BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this.height = height;
    this.width = width;
    this.channels = channels;
    this.labelGenerator = labelGenerator;
    this.labelMultiGenerator = labelMultiGenerator;
    this.imageTransform = imageTransform;
    this.appendLabel = (labelGenerator != null || labelMultiGenerator != null);
}
 
Example #5
Source File: JacksonRecordReader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 
Example #6
Source File: LFWLoader.java    From DataVec with Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, int numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        e.printStackTrace();
    }
    return recordReader;
}
 
Example #7
Source File: JacksonRecordReader.java    From DataVec with Apache License 2.0 5 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator, int labelPosition) {
    this.selection = selection;
    this.mapper = mapper;
    this.shuffle = shuffle;
    this.rngSeed = rngSeed;
    if (shuffle)
        r = new Random(rngSeed);
    this.labelGenerator = labelGenerator;
    this.labelPosition = labelPosition;
}
 
Example #8
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 
Example #9
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, long numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        log.error("",e);
    }
    return recordReader;
}
 
Example #10
Source File: FileBatchRecordReaderTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testCsv() throws Exception {
    File extractedSourceDir = testDir.newFolder();
    new ClassPathResource("datavec-data-image/testimages").copyDirectory(extractedSourceDir);
    File baseDir = testDir.newFolder();


    List<File> c = new ArrayList<>(FileUtils.listFiles(extractedSourceDir, null, true));
    assertEquals(6, c.size());

    Collections.sort(c, new Comparator<File>() {
        @Override
        public int compare(File o1, File o2) {
            return o1.getPath().compareTo(o2.getPath());
        }
    });


    FileBatch fb = FileBatch.forFiles(c);
    File saveFile = new File(baseDir, "saved.zip");
    fb.writeAsZip(saveFile);
    fb = FileBatch.readFromZip(saveFile);

    PathLabelGenerator labelMaker = new ParentPathLabelGenerator();
    ImageRecordReader rr = new ImageRecordReader(32, 32, 1, labelMaker);
    rr.setLabels(Arrays.asList("class0", "class1"));
    FileBatchRecordReader fbrr = new FileBatchRecordReader(rr, fb);


    NativeImageLoader il = new NativeImageLoader(32, 32, 1);
    for( int test=0; test<3; test++) {
        for (int i = 0; i < 6; i++) {
            assertTrue(fbrr.hasNext());
            List<Writable> next = fbrr.next();
            assertEquals(2, next.size());

            INDArray exp;
            switch (i){
                case 0:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/0.jpg"));
                    break;
                case 1:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/1.png"));
                    break;
                case 2:
                    exp = il.asMatrix(new File(extractedSourceDir, "class0/2.jpg"));
                    break;
                case 3:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/A.jpg"));
                    break;
                case 4:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/B.png"));
                    break;
                case 5:
                    exp = il.asMatrix(new File(extractedSourceDir, "class1/C.jpg"));
                    break;
                default:
                    throw new RuntimeException();
            }
            Writable expLabel = (i < 3 ? new IntWritable(0) : new IntWritable(1));

            assertEquals(((NDArrayWritable)next.get(0)).get(), exp);
            assertEquals(expLabel, next.get(1));
        }
        assertFalse(fbrr.hasNext());
        assertTrue(fbrr.resetSupported());
        fbrr.reset();
    }
}
 
Example #11
Source File: BaseImageRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}
 
Example #12
Source File: BaseImageRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
                             ImageTransform imageTransform) {
    this(height, width, channels, labelGenerator, null, imageTransform);
}
 
Example #13
Source File: BaseImageRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
protected BaseImageRecordReader(long height, long width, long channels, PathLabelGenerator labelGenerator,
                                PathMultiLabelGenerator labelMultiGenerator, ImageTransform imageTransform) {
    this(height, width, channels, true, labelGenerator, labelMultiGenerator, imageTransform);
}
 
Example #14
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, new long[] {height, width, channels},
                    useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator, train, splitTrainTest, rng);
}
 
Example #15
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, int[] imgDim, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
                    train, splitTrainTest, rng);
}
 
Example #16
Source File: LFWLoader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public RecordReader getRecordReader(long batchSize, long numExamples, long[] imgDim, PathLabelGenerator labelGenerator,
                boolean train, double splitTrainTest, Random rng) {
    return getRecordReader(numExamples, batchSize, imgDim, useSubset ? SUB_NUM_LABELS : NUM_LABELS, labelGenerator,
                    train, splitTrainTest, rng);
}
 
Example #17
Source File: TestImageRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testImageRecordReaderRegression() throws Exception {

    PathLabelGenerator regressionLabelGen = new TestRegressionLabelGen();

    ImageRecordReader rr = new ImageRecordReader(28, 28, 3, regressionLabelGen);

    File rootDir = testDir.newFolder();
    new ClassPathResource("datavec-data-image/testimages/").copyDirectory(rootDir);
    FileSplit fs = new FileSplit(rootDir);
    rr.initialize(fs);
    URI[] arr = fs.locations();

    assertTrue(rr.getLabels() == null || rr.getLabels().isEmpty());

    List<Writable> expLabels = new ArrayList<>();
    for(URI u : arr){
        String path = u.getPath();
        expLabels.add(testLabel(path.substring(path.length()-5, path.length())));
    }

    int count = 0;
    while(rr.hasNext()){
        List<Writable> l = rr.next();

        assertEquals(2, l.size());
        assertEquals(expLabels.get(count), l.get(1));

        count++;
    }
    assertEquals(6, count);

    //Test batch ops:
    rr.reset();

    List<List<Writable>> b1 = rr.next(3);
    List<List<Writable>> b2 = rr.next(3);
    assertFalse(rr.hasNext());

    NDArrayRecordBatch b1a = (NDArrayRecordBatch)b1;
    NDArrayRecordBatch b2a = (NDArrayRecordBatch)b2;
    assertEquals(2, b1a.getArrays().size());
    assertEquals(2, b2a.getArrays().size());

    NDArrayWritable l1 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(0).toDouble(),
            expLabels.get(1).toDouble(), expLabels.get(2).toDouble()}, new long[]{3,1}, DataType.FLOAT));
    NDArrayWritable l2 = new NDArrayWritable(Nd4j.create(new double[]{expLabels.get(3).toDouble(),
            expLabels.get(4).toDouble(), expLabels.get(5).toDouble()}, new long[]{3,1}, DataType.FLOAT));

    INDArray act1 = b1a.getArrays().get(1);
    INDArray act2 = b2a.getArrays().get(1);
    assertEquals(l1.get(), act1);
    assertEquals(l2.get(), act2);
}
 
Example #18
Source File: ImageRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending no labels. */
public ImageRecordReader(int height, int width, int channels) {
    super(height, width, channels, (PathLabelGenerator) null);
}
 
Example #19
Source File: JacksonRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public JacksonRecordReader(FieldSelection selection, ObjectMapper mapper, boolean shuffle, long rngSeed,
                PathLabelGenerator labelGenerator) {
    this(selection, mapper, shuffle, rngSeed, labelGenerator, -1);
}
 
Example #20
Source File: BalancedPathFilter.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, 0)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator) {
    this(random, extensions, labelGenerator, 0, 0, 0, 0);
}
 
Example #21
Source File: BalancedPathFilter.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPathsPerLabel) {
    this(random, null, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 
Example #22
Source File: BalancedPathFilter.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator,
                long maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, 0, 0, maxPathsPerLabel);
}
 
Example #23
Source File: BalancedPathFilter.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, PathLabelGenerator labelGenerator, long maxPaths, long maxLabels,
                long maxPathsPerLabel) {
    this(random, null, labelGenerator, maxPaths, maxLabels, 0, maxPathsPerLabel);
}
 
Example #24
Source File: BalancedPathFilter.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Calls {@code this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel)}. */
public BalancedPathFilter(Random random, String[] extensions, PathLabelGenerator labelGenerator, long maxLabels,
                long maxPathsPerLabel) {
    this(random, extensions, labelGenerator, 0, maxLabels, 0, maxPathsPerLabel);
}
 
Example #25
Source File: LFWDataSetIterator.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Loads images with given  batchSize, numExamples, imgDim, numLabels, useSubset, train, splitTrainTest & Random returned by the generator. */
public LFWDataSetIterator(int batchSize, int numExamples, int[] imgDim, int numLabels, boolean useSubset,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    this(batchSize, numExamples, imgDim, numLabels, useSubset, labelGenerator, train, splitTrainTest, null, rng);
}
 
Example #26
Source File: ImageRecordReader.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator.<br>
 * If {@code nchw_channels_first == true} output format is NCHW (channels first) - [numExamples, channels, height, width]<br>
 * If {@code nchw_channels_first == false} output format is NHWC (channels last) - [numExamples, height, width, channels]<br>
 */
public ImageRecordReader(long height, long width, long channels, boolean nchw_channels_first, PathLabelGenerator labelGenerator,
                         ImageTransform imageTransform) {
    super(height, width, channels, nchw_channels_first, labelGenerator, null, imageTransform);
}
 
Example #27
Source File: ImageRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    super(height, width, channels, labelGenerator);
}
 
Example #28
Source File: ImageRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up
/** Loads images with given height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator,
                ImageTransform imageTransform) {
    super(height, width, channels, labelGenerator, imageTransform);
}
 
Example #29
Source File: ImageRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up
/** Loads images with given  height, width, and channels, appending labels returned by the generator. */
public ImageRecordReader(int height, int width, PathLabelGenerator labelGenerator) {
    super(height, width, 1, labelGenerator);
}
 
Example #30
Source File: BaseImageRecordReader.java    From DataVec with Apache License 2.0 4 votes vote down vote up
public BaseImageRecordReader(int height, int width, int channels, PathLabelGenerator labelGenerator) {
    this(height, width, channels, labelGenerator, null);
}