org.datavec.api.records.writer.RecordWriter Java Examples

/**
 * Write all values from the specified record reader to the specified record writer.
 * Optionally, close the record writer on completion
 *
 * @param reader Record reader (source of data)
 * @param writer Record writer (location to write data)
 * @param closeOnCompletion if true: close the record writer once complete, via {@link RecordWriter#close()}
 * @throws IOException If underlying reader/writer throws an exception
 */
public static void convert(RecordReader reader, RecordWriter writer, boolean closeOnCompletion) throws IOException {

    if(!reader.hasNext()){
        throw new UnsupportedOperationException("Cannot convert RecordReader: reader has no next element");
    }

    while(reader.hasNext()){
        writer.write(reader.next());
    }

    if(closeOnCompletion){
        writer.close();
    }
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    try {
        //return new LineRecordWriter(new File(outputPath));
        return new SVMLightRecordWriter();
    } catch (Exception e) {
        throw new DataVecException(e);
    }
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    try {
        //return new LineRecordWriter(new File(outputPath));
        return new SVMLightRecordWriter();
    } catch (Exception e) {
        throw new DataVecException(e);
    }
}
 

/**
 * Write all values from the specified record reader to the specified record writer.
 * Optionally, close the record writer on completion
 *
 * @param reader Record reader (source of data)
 * @param writer Record writer (location to write data)
 * @param closeOnCompletion if true: close the record writer once complete, via {@link RecordWriter#close()}
 * @throws IOException If underlying reader/writer throws an exception
 */
public static void convert(RecordReader reader, RecordWriter writer, boolean closeOnCompletion) throws IOException {

    if(!reader.hasNext()){
        throw new UnsupportedOperationException("Cannot convert RecordReader: reader has no next element");
    }

    while(reader.hasNext()){
        writer.write(reader.next());
    }

    if(closeOnCompletion){
        writer.close();
    }
}
 

@Override
public void recordWrite(RecordWriter writer, Object record) {
    invoke();
    log.info("Writing " + record);
}
 

@Override
public void recordWrite(RecordWriter writer, Object record) {
    invoke();
    log.info("Writing " + record);
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    return new CSVRecordWriter();
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    return new LineRecordWriter();
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    RecordWriter writer = new LibSvmRecordWriter();
    writer.setConf(conf);
    return writer;
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    return null;
}
 

@Override
public void recordWrite(RecordWriter writer, Object record) {
    this.listener.recordWrite(writer, record);
    this.count++;
}
 

public static void main(String[] args) throws Exception {
    try {
        int numClasses = 2;
        int batchSize = 8;

        File file = new File("Path/to/titanic.csv-file");
        RecordReader recordReader = new CSVRecordReader(1,',');
        recordReader.initialize(new FileSplit(file));
        // WritableConverter writableConverter = new SelfWritableConverter();

        Schema schema = new Schema.Builder()
                .addColumnInteger("Survived")
                .addColumnCategorical("Pclass", Arrays.asList("1","2","3"))
                .addColumnString("Name")
                .addColumnCategorical("Sex", Arrays.asList("male","female"))
                .addColumnsInteger("Age","Siblings/Spouses Aboard","Parents/Children Aboard")
                .addColumnDouble("Fare")
                .build();
        TransformProcess transformProcess = new TransformProcess.Builder(schema)
                .removeColumns("Name","Fare")
                .categoricalToInteger("Sex")
                .categoricalToOneHot("Pclass")
                .removeColumns("Pclass[1]")
                .build();

        List<List<Writable>> outputData = new ArrayList<>();

        RecordWriter recordWriter = new CSVRecordWriter();
        Partitioner partitioner = new NumberOfRecordsPartitioner();
        recordWriter.initialize(new FileSplit(new File("/Path/To/LocalExecuteExample.csv/file")),partitioner);

        while(recordReader.hasNext()){
            outputData.add(recordReader.next());
        }
        List<List<Writable>> transformedOutput=LocalTransformExecutor.execute(outputData,transformProcess);
        recordWriter.writeBatch(transformedOutput);
        recordWriter.close();
    } catch (IllegalArgumentException e) {
        System.out.println("Please provide proper file paths for titanic.csv & fle in place of: Path/to/titanic.csv-file && /Path/To/LocalExecuteExample.csv");
        System.out.println("You need to create an empty CSV file and mention the file path in place of /Path/To/LocalExecuteExample.csv");
    }
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    return new CSVRecordWriter();
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    String outputPath = conf.get(OutputFormat.OUTPUT_PATH, ".");
    return new LineRecordWriter();
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    RecordWriter writer = new LibSvmRecordWriter();
    writer.setConf(conf);
    return writer;
}
 

@Override
public RecordWriter createWriter(Configuration conf) throws DataVecException {
    return null;
}
 

@Override
public void recordWrite(RecordWriter writer, Object record) {
    this.listener.recordWrite(writer, record);
    this.count++;
}
 

public static void main(String[] args) throws Exception {
    try {
        int numClasses = 2;
        int batchSize = 8;

        File file = new File("Path/to/titanic.csv-file");
        RecordReader recordReader = new CSVRecordReader(1,',');
        recordReader.initialize(new FileSplit(file));
        // WritableConverter writableConverter = new SelfWritableConverter();

        Schema schema = new Schema.Builder()
                .addColumnInteger("Survived")
                .addColumnCategorical("Pclass", Arrays.asList("1","2","3"))
                .addColumnString("Name")
                .addColumnCategorical("Sex", Arrays.asList("male","female"))
                .addColumnsInteger("Age","Siblings/Spouses Aboard","Parents/Children Aboard")
                .addColumnDouble("Fare")
                .build();
        TransformProcess transformProcess = new TransformProcess.Builder(schema)
                .removeColumns("Name","Fare")
                .categoricalToInteger("Sex")
                .categoricalToOneHot("Pclass")
                .removeColumns("Pclass[1]")
                .build();

        List<List<Writable>> outputData = new ArrayList<>();

        RecordWriter recordWriter = new CSVRecordWriter();
        Partitioner partitioner = new NumberOfRecordsPartitioner();
        recordWriter.initialize(new FileSplit(new File("/Path/To/LocalExecuteExample.csv/file")),partitioner);

        while(recordReader.hasNext()){
            outputData.add(recordReader.next());
        }
        List<List<Writable>> transformedOutput=LocalTransformExecutor.execute(outputData,transformProcess);
        recordWriter.writeBatch(transformedOutput);
        recordWriter.close();
    } catch (IllegalArgumentException e) {
        System.out.println("Please provide proper file paths for titanic.csv & fle in place of: Path/to/titanic.csv-file && /Path/To/LocalExecuteExample.csv");
        System.out.println("You need to create an empty CSV file and mention the file path in place of /Path/To/LocalExecuteExample.csv");
    }
}
 

/**
 * Event listener for each record to be written.
 * @param writer the record writer
 * @param record in raw format (Collection, File, String, Writable, etc)
 */
void recordWrite(RecordWriter writer, Object record);
 

/**
 * Write all values from the specified record reader to the specified record writer.
 * Closes the record writer on completion
 *
 * @param reader Record reader (source of data)
 * @param writer Record writer (location to write data)
 * @throws IOException If underlying reader/writer throws an exception
 */
public static void convert(RecordReader reader, RecordWriter writer) throws IOException {
    convert(reader, writer, true);
}
 

/**
 * Create a record writer
 * @return the created writer
 */
RecordWriter createWriter(Configuration conf) throws DataVecException;
 

/**
 * Create a record writer
 * @return the created writer
 */
RecordWriter createWriter(Configuration conf) throws DataVecException;
 

/**
 * Write all values from the specified record reader to the specified record writer.
 * Closes the record writer on completion
 *
 * @param reader Record reader (source of data)
 * @param writer Record writer (location to write data)
 * @throws IOException If underlying reader/writer throws an exception
 */
public static void convert(RecordReader reader, RecordWriter writer) throws IOException {
    convert(reader, writer, true);
}
 

/**
 * Event listener for each record to be written.
 * @param writer the record writer
 * @param record in raw format (Collection, File, String, Writable, etc)
 */
void recordWrite(RecordWriter writer, Object record);
 

/**
 *
 * @param uri destination for saving model
 * @return record writer instance
 * @throws Exception
 */

RecordWriter create(URI uri) throws Exception;
 

/**
 *
 * @param uri destination for saving model
 * @return record writer instance
 * @throws Exception
 */

RecordWriter create(URI uri) throws Exception;