org.embulk.config.ConfigSource Java Examples

@Test
public void checkExtraConfigurations()
        throws NoSuchMethodException, InvocationTargetException, IllegalAccessException
{
    ConfigSource map = Exec.newConfigSource()
            .set("foo", "bar");

    ConfigSource config = Exec.newConfigSource()
            .set("path_prefix", "test")
            .setNested("extra_configurations", map);

    ParquetOutputPlugin.PluginTask task = config.loadConfig(ParquetOutputPlugin.PluginTask.class);

    Map<String, String> extra = task.getExtraConfigurations();
    assertTrue(extra.containsKey("foo"));
    assertEquals("bar", extra.get("foo"));

    ParquetOutputPlugin plugin = new ParquetOutputPlugin();
    Method method = ParquetOutputPlugin.class.getDeclaredMethod("createConfiguration", Map.class, List.class);
    method.setAccessible(true);
    Configuration conf = (Configuration) method.invoke(plugin, extra, new ArrayList<String>());
    assertEquals("bar", conf.get("foo"));
}
 

@Test
public void writeNils()
        throws Exception
{
    SchemaConfig schema = schema(
            column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
            column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
    ConfigSource config = this.config.deepCopy().set("columns", schema);

    transaction(config, fileInput(
            "{}",
            "{\"_c0\":null,\"_c1\":null,\"_c2\":null}",
            "{\"_c3\":null,\"_c4\":null,\"_c5\":null}",
            "{}"
    ));

    List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
    assertEquals(4, records.size());

    for (Object[] record : records) {
        for (int i = 0; i < 6; i++) {
            assertNull(record[i]);
        }
    }
}
 

@Test
public void throwDataException()
        throws Exception
{
    SchemaConfig schema = schema(
            column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
            column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
    ConfigSource config = this.config.deepCopy().set("columns", schema).set("stop_on_invalid_record", true);

    try {
        transaction(config, fileInput(
                "\"not_map_value\""
        ));
        fail();
    }
    catch (Throwable t) {
        assertTrue(t instanceof DataException);
    }
}
 

@Override
public void transaction(ConfigSource configSource, Control control)
{
    PluginTask task = configSource.loadConfig(PluginTask.class);

    if (! task.getColumnOptions().isEmpty()) {
        log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true).");
    }

    SchemaConfig schemaConfig = getSchemaConfig(task);
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
        ColumnConfig columnConfig = schemaConfig.getColumn(i);
        Type type = getType(task, columnConfig);
        columns.add(new Column(i, columnConfig.getName(), type));
    }
    control.run(task.dump(), new Schema(columns.build()));
}
 

@Test
public void skipRecords()
        throws Exception
{
    SchemaConfig schema = schema(
            column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
            column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
    ConfigSource config = this.config.deepCopy().set("columns", schema);

    transaction(config, fileInput(
            "[]",
            "\"embulk\"",
            "10",
            "true",
            "false",
            "null",
            " "
    ));

    List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
    assertEquals(0, records.size());
}
 

@Override
public ConfigDiff transaction(ConfigSource config, int taskCount,
        Control control)
{
    PluginTask task = config.loadConfig(PluginTask.class);

    validateSequenceFormat(task);

    return resume(task.dump(), taskCount, control);
}
 

@Override
public void transaction(final ConfigSource config, final Schema inputSchema,
        final FilterPlugin.Control control)
{
    PluginTask task = config.loadConfig(PluginTask.class);

    configure(task);
    Schema outputSchema = buildOutputSchema(task, inputSchema);

    control.run(task.dump(), outputSchema);
}
 

@Test(expected = ConfigException.class)
public void checkColumnsRequired()
{
    ConfigSource config = Exec.newConfigSource();

    config.loadConfig(ParquetOutputPlugin.PluginTask.class);
}
 

@Test
public void checkDefaultValues()
{
    ConfigSource config = Exec.newConfigSource()
            .set("path_prefix", "test");

    ParquetOutputPlugin.PluginTask task = config.loadConfig(ParquetOutputPlugin.PluginTask.class);
    assertEquals(".parquet", task.getFileNameExtension());
    assertEquals(".%03d", task.getSequenceFormat());
    assertEquals(134217728, task.getBlockSize());
    assertEquals(1048576, task.getPageSize());
    assertEquals("UNCOMPRESSED", task.getCompressionCodec());
    assertFalse(task.getOverwrite());
    assertEquals("false", task.getSignature());
}
 

private void transaction(ConfigSource config, final FileInput input)
{
    plugin.transaction(config, new ParserPlugin.Control()
    {
        @Override
        public void run(TaskSource taskSource, Schema schema)
        {
            plugin.run(taskSource, schema, input, output);
        }
    });
}
 

@Test
public void useColumnOptions()
        throws Exception
{

    SchemaConfig schema = schema(
            column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE));
    File yamlFile = getResourceFile("use_column_options.yml");
    ConfigSource config = getConfigFromYamlFile(yamlFile);

    transaction(config, fileInput(
            "{\"_c0\":\"true\",\"_c1\":\"10\",\"_c2\":\"0.1\"}",
            "{\"_c0\":\"false\",\"_c1\":\"-10\",\"_c2\":\"1.0\"}"
    ));

    List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
    assertEquals(2, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(true, record[0]);
        assertEquals(10L, record[1]);
        assertEquals(0.1, (Double) record[2], 0.0001);
    }
    {
        record = records.get(1);
        assertEquals(false, record[0]);
        assertEquals(-10L, record[1]);
        assertEquals(1.0, (Double) record[2], 0.0001);
    }
}
 

private PluginTask taskFromYamlString(String... lines)
{
    StringBuilder builder = new StringBuilder();
    for (String line : lines) {
        builder.append(line).append("\n");
    }
    String yamlString = builder.toString();

    ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
    ConfigSource config = loader.fromYamlString(yamlString);
    return config.loadConfig(PluginTask.class);
}
 

@Test(expected = ConfigException.class)
public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
{
    ConfigSource config = configFromYamlString(
            "type: column",
            "columns:",
            "- {name: a}",
            "drop_columns:",
            "- {name: a}");
    Schema inputSchema = schema(
            new Column(0, "a", STRING),
            new Column(1, "b", STRING));

    transaction(config, inputSchema);
}
 

private void transaction(ConfigSource config, Schema inputSchema)
{
    plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
        }
    });
}
 

private ConfigSource configFromYamlString(String... lines)
{
    StringBuilder builder = new StringBuilder();
    for (String line : lines) {
        builder.append(line).append("\n");
    }
    String yamlString = builder.toString();

    ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
    return loader.fromYamlString(yamlString);
}
 

private PluginTask taskFromYamlString(String... lines)
{
    StringBuilder builder = new StringBuilder();
    for (String line : lines) {
        builder.append(line).append("\n");
    }
    String yamlString = builder.toString();

    ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
    ConfigSource config = loader.fromYamlString(yamlString);
    return config.loadConfig(PluginTask.class);
}
 

private ConfigSource config()
{
    return runtime.getExec().newConfigSource();
}
 

private ConfigSource config()
{
    return runtime.getExec().newConfigSource();
}
 

private ConfigSource getConfigFromYamlFile(File yamlFile)
        throws IOException
{
    ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
    return loader.fromYamlFile(yamlFile);
}
 

private ColumnConfig column(String name, Type type, ConfigSource option)
{
    return new ColumnConfig(name, type, option);
}
 

private PluginTask taskFromYamlString(String... lines)
{
    ConfigSource config = configFromYamlString(lines);
    return config.loadConfig(PluginTask.class);
}
 

private ConfigSource config()
{
    return runtime.getExec().newConfigSource();
}