Java Code Examples for org.datavec.api.conf.Configuration#setBoolean()

The following examples show how to use org.datavec.api.conf.Configuration#setBoolean() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SVMLightRecordWriterTest.java From deeplearning4j with Apache License 2.0

6 votes

@Test(expected = NumberFormatException.class)
public void nonBinaryMultilabel() throws Exception {
    List<Writable> record = Arrays.asList((Writable) new IntWritable(0),
            new IntWritable(1),
            new IntWritable(2));
    File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1);
        configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }
}

Example 2

Source File: SVMLightRecordWriterTest.java From DataVec with Apache License 2.0

6 votes

@Test
public void testMultilabelRecord() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 9);
    configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);

    Configuration configReader = new Configuration();
    configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    configReader.setBoolean(SVMLightRecordReader.MULTILABEL, true);
    configReader.setInt(SVMLightRecordReader.NUM_LABELS, 4);
    configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);

    File inputFile = new ClassPathResource("svmlight/multilabel.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}

Example 3

Source File: SVMLightRecordWriterTest.java From deeplearning4j with Apache License 2.0

6 votes

@Test
public void testZeroBasedIndexing() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_INDEXING, true);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 10);
    configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);

    Configuration configReader = new Configuration();
    configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 11);
    configReader.setBoolean(SVMLightRecordReader.MULTILABEL, true);
    configReader.setInt(SVMLightRecordReader.NUM_LABELS, 5);

    File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}

Example 4

Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testZeroIndexFeatureWithoutUsingZeroIndexing() throws Exception {
    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/zeroIndexFeature.txt").getFile()));
    rr.next();
}

Example 5

Source File: LibSvmRecordWriterTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test
public void testNDArrayWritablesMultilabel() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 0);
    arr3.putScalar(1, 1);
    arr3.putScalar(2, 0);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
            new NDArrayWritable(arr2),
            new IntWritable(2),
            new DoubleWritable(3),
            new NDArrayWritable(arr3),
            new DoubleWritable(1));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0";

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}

Example 6

Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testZeroIndexFeatureWithoutUsingZeroIndexing() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexFeature.txt").getFile()));
    rr.next();
}

Example 7

Source File: LibSvmRecordWriterTest.java From DataVec with Apache License 2.0

5 votes

@Test
public void testNDArrayWritablesZeroIndex() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 0);
    arr3.putScalar(1, 1);
    arr3.putScalar(2, 0);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
            new NDArrayWritable(arr2),
            new IntWritable(2),
            new DoubleWritable(3),
            new NDArrayWritable(arr3),
            new DoubleWritable(1));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "1,3 0:1.0 1:11.0 2:12.0 3:2.0 4:3.0";

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true); // NOT STANDARD!
        configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD!
        configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}

Example 8

Source File: LibSvmRecordWriterTest.java From DataVec with Apache License 2.0

5 votes

@Test
public void testMultioutputRecord() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9);

    Configuration configReader = new Configuration();
    configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);

    File inputFile = new ClassPathResource("svmlight/multioutput.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}

Example 9

Source File: SVMLightRecordReaderTest.java From DataVec with Apache License 2.0

5 votes

@Test
public void testNoAppendLabel() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 7 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5)));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO));
    // 33
    correct.put(2, Arrays.asList(ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO));

    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 10

Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test(expected = UnsupportedOperationException.class)
public void testInconsistentNumMultiabelsException() throws Exception {
    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.MULTILABEL, false);
    config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile()));
    while (rr.hasNext())
        rr.next();
}

Example 11

Source File: SVMLightRecordWriterTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test
public void testMultioutputRecord() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 9);

    Configuration configReader = new Configuration();
    configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);

    File inputFile = new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}

Example 12

Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testZeroIndexLabelWithoutUsingZeroIndexing() throws Exception {
    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    config.setBoolean(LibSvmRecordReader.MULTILABEL, true);
    config.setInt(LibSvmRecordReader.NUM_LABELS, 2);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/zeroIndexLabel.txt").getFile()));
    rr.next();
}

Example 13

Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testZeroIndexLabelWithoutUsingZeroIndexing() throws Exception {
    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    config.setBoolean(LibSvmRecordReader.MULTILABEL, true);
    config.setInt(LibSvmRecordReader.NUM_LABELS, 2);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile()));
    rr.next();
}

Example 14

Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0

5 votes

@Test
public void testNoAppendLabel() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 7 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
            ZERO, new DoubleWritable(2),
            ZERO, new DoubleWritable(3),
            ZERO, new DoubleWritable(4),
            ZERO, new DoubleWritable(5)));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
            ZERO, ZERO,
            ZERO, new DoubleWritable(6.6),
            ZERO, new DoubleWritable(80),
            ZERO, ZERO));
    // 33
    correct.put(2, Arrays.asList(ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO));

    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    config.setBoolean(SVMLightRecordReader.APPEND_LABEL, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 15

Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void testNoLabel() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    //  2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
            ZERO, new DoubleWritable(2),
            ZERO, new DoubleWritable(3),
            ZERO, new DoubleWritable(4),
            ZERO, new DoubleWritable(5)));
    //  qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
            ZERO, ZERO,
            ZERO, new DoubleWritable(6.6),
            ZERO, new DoubleWritable(80),
            ZERO, ZERO));
    //  1:1.0
    correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO));
    //
    correct.put(3, Arrays.asList(ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO));

    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/noLabels.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 16

Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0

4 votes

@Test
public void testMultilabelRecord() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 1,3 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5),
                                LABEL_ONE, LABEL_ZERO,
                                LABEL_ONE, LABEL_ZERO));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO,
                                LABEL_ZERO, LABEL_ONE,
                                LABEL_ZERO, LABEL_ZERO));
    // 1,2,4
    correct.put(2, Arrays.asList(ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                LABEL_ONE, LABEL_ONE,
                                LABEL_ZERO, LABEL_ONE));
    //  1:1.0
    correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));
    //
    correct.put(4, Arrays.asList(ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));

    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    config.setBoolean(LibSvmRecordReader.MULTILABEL, true);
    config.setInt(LibSvmRecordReader.NUM_LABELS, 4);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/multilabel.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 17

Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void testZeroBasedIndexing() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 1,3 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO,
                                ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5),
                                LABEL_ZERO,
                                LABEL_ONE, LABEL_ZERO,
                                LABEL_ONE, LABEL_ZERO));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(ZERO,
                                new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO,
                                LABEL_ZERO,
                                LABEL_ZERO, LABEL_ONE,
                                LABEL_ZERO, LABEL_ZERO));
    // 1,2,4
    correct.put(2, Arrays.asList(ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                LABEL_ZERO,
                                LABEL_ONE, LABEL_ONE,
                                LABEL_ZERO, LABEL_ONE));
    //  1:1.0
    correct.put(3, Arrays.asList(ZERO,
            new DoubleWritable(1.0), ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));
    //
    correct.put(4, Arrays.asList(ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));

    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    // Zero-based indexing is default
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD!
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 11);
    config.setBoolean(LibSvmRecordReader.MULTILABEL, true);
    config.setInt(LibSvmRecordReader.NUM_LABELS, 5);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 18

Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void testZeroBasedIndexing() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 1,3 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO,
                                ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5),
                                LABEL_ZERO,
                                LABEL_ONE, LABEL_ZERO,
                                LABEL_ONE, LABEL_ZERO));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(ZERO,
                                new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO,
                                LABEL_ZERO,
                                LABEL_ZERO, LABEL_ONE,
                                LABEL_ZERO, LABEL_ZERO));
    // 1,2,4
    correct.put(2, Arrays.asList(ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                LABEL_ZERO,
                                LABEL_ONE, LABEL_ONE,
                                LABEL_ZERO, LABEL_ONE));
    //  1:1.0
    correct.put(3, Arrays.asList(ZERO,
            new DoubleWritable(1.0), ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));
    //
    correct.put(4, Arrays.asList(ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            ZERO, ZERO,
            LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO,
            LABEL_ZERO, LABEL_ZERO));

    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    // Zero-based indexing is default
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD!
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 11);
    config.setBoolean(SVMLightRecordReader.MULTILABEL, true);
    config.setInt(SVMLightRecordReader.NUM_LABELS, 5);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 19

Source File: SVMLightRecordReaderTest.java From DataVec with Apache License 2.0

4 votes

@Test
public void testMultioutputRecord() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 7 2.45,9 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5),
                                new IntWritable(7), new DoubleWritable(2.45),
                                new IntWritable(9)));
    // 2,3,4 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO,
                                new IntWritable(2), new IntWritable(3),
                                new IntWritable(4)));
    // 33,32.0,31.9
    correct.put(2, Arrays.asList(ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                new IntWritable(33), new DoubleWritable(32.0),
                                new DoubleWritable(31.9)));

    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/multioutput.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}

Example 20

Source File: DataIteratorConstructor.java From scava with Eclipse Public License 2.0

4 votes

private void buildDataIterator(Configuration configuration, FileDataSet fileDataSet, VasttextTextVectorizer vectorizer) throws IOException, InterruptedException
{
	Builder vasttextDataIteratorBuilder = new VasttextDataIterator.Builder(configuration.getInt(batchSize, 32));
	
	configuration.setBoolean(VasttextTextFileReader.LABELLED, fileDataSet.isLabelled());
	configuration.setBoolean(VasttextTextFileReader.MULTILABEL, fileDataSet.isMultiLabel());
	
	VasttextTextFileReader textFileReader = new VasttextTextFileReader();
	if(vectorizer!=null)
		textFileReader.setVastTextTextVectorizer(vectorizer);
	textFileReader.initialize(configuration, new FileSplit(fileDataSet.getTextFilePath().toFile()));
	
	multilabel=fileDataSet.isMultiLabel();
	textFeaturesSize=textFileReader.getTextFeaturesSize();
	labels=textFileReader.getLabels();
	
	if(labels!=null && storeVectorizer)
	{
		this.vectorizer=textFileReader.getVasttextTextVectorizer();
	}
	else
	{
		storeVectorizer=false;
		this.vectorizer=null;
	}
	
	vasttextDataIteratorBuilder.addReader("textReader", textFileReader);
	if(labels!=null)
		vasttextDataIteratorBuilder.addOutput("textReader", labels.size());
	vasttextDataIteratorBuilder.addInput("textReader");
	
	if(fileDataSet.getExtraFilePath()!=null)
	{
		configuration.setInt(VasttextExtraFileReader.FEATURES, fileDataSet.numericFeaturesSize());
		
		VasttextExtraFileReader extraFileReader = new VasttextExtraFileReader(textFileReader.getLinesDeleted());
		extraFileReader.initialize(configuration, new FileSplit(fileDataSet.getExtraFilePath().toFile()));
		
		numericFeaturesSize=fileDataSet.numericFeaturesSize();
		
		vasttextDataIteratorBuilder.addReader("extraReader", extraFileReader);
		vasttextDataIteratorBuilder.addInput("extraReader");
	}

	dataIterator = vasttextDataIteratorBuilder.build();
	
}