org.embulk.config.TaskSource Java Examples

The following examples show how to use org.embulk.config.TaskSource. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Override
public PageOutput open(final TaskSource taskSource, final Schema inputSchema,
        final Schema outputSchema, final PageOutput output)
{
    final PluginTask task = taskSource.loadTask(PluginTask.class);

    return new PageOutput() {
        private PageReader pageReader = new PageReader(inputSchema);
        private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
        private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);

        @Override
        public void finish()
        {
            pageBuilder.finish();
        }

        @Override
        public void close()
        {
            pageBuilder.close();
        }

        @Override
        public void add(Page page)
        {
            pageReader.setPage(page);

            while (pageReader.nextRecord()) {
                outputSchema.visitColumns(visitor);
                pageBuilder.addRecord();
            }
        }
    };
}
Example #2
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
private void transaction(ConfigSource config, Schema inputSchema)
{
    plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
        }
    });
}
Example #3
Source File: TestJsonlParserPlugin.java    From embulk-parser-jsonl with MIT License 5 votes vote down vote up 
private void transaction(ConfigSource config, final FileInput input)
{
    plugin.transaction(config, new ParserPlugin.Control()
    {
        @Override
        public void run(TaskSource taskSource, Schema schema)
        {
            plugin.run(taskSource, schema, input, output);
        }
    });
}
Example #4
Source File: S3FileOutputPlugin.java    From embulk-output-s3 with MIT License 5 votes vote down vote up 
@Override
public ConfigDiff resume(TaskSource taskSource, int taskCount,
        Control control)
{
    control.run(taskSource);
    return Exec.newConfigDiff();
}
Example #5
Source File: S3FileOutputPlugin.java    From embulk-output-s3 with MIT License 5 votes vote down vote up 
@Override
public TransactionalFileOutput open(TaskSource taskSource, int taskIndex)
{
    PluginTask task = taskSource.loadTask(PluginTask.class);

    return new S3FileOutput(task, taskIndex);
}
Example #6
Source File: JsonlParserPlugin.java    From embulk-parser-jsonl with MIT License 4 votes vote down vote up 
@Override
public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output)
{
    PluginTask task = taskSource.loadTask(PluginTask.class);

    setColumnNameValues(schema);

    final SchemaConfig schemaConfig = getSchemaConfig(task);
    final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, schemaConfig);
    final LineDecoder decoder = newLineDecoder(input, task);
    final JsonParser jsonParser = newJsonParser();
    final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();

    try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
        ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);

        while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3
            lineNumber = 0;

            while ((line = decoder.poll()) != null) {
                lineNumber++;

                try {
                    Value value = jsonParser.parse(line);

                    if (!value.isMapValue()) {
                        throw new JsonRecordValidateException("Json string is not representing map value.");
                    }

                    final Map<Value, Value> record = value.asMapValue().map();
                    for (Column column : schema.getColumns()) {
                        Value v = record.get(getColumnNameValue(column));
                        visitor.setValue(v);
                        column.visit(visitor);
                    }

                    pageBuilder.addRecord();
                }
                catch (JsonRecordValidateException | JsonParseException e) {
                    if (stopOnInvalidRecord) {
                        throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, line), e);
                    }
                    log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), line));
                }
            }
        }

        pageBuilder.finish();
    }
}
Example #7
Source File: S3FileOutputPlugin.java    From embulk-output-s3 with MIT License 4 votes vote down vote up 
@Override
public void cleanup(TaskSource taskSource, int taskCount,
        List<TaskReport> successTaskReports)
{
}