Java Code Examples for org.msgpack.value.Value

The following examples show how to use org.msgpack.value.Value. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: embulk-filter-column   Source File: JsonVisitor.java    License: MIT License 6 votes vote down vote up
public Value visit(String rootPath, Value value)
{
    if (! shouldVisit(rootPath)) {
        return value;
    }
    if (value == null) {
        return null;
    }
    else if (value.isArrayValue()) {
        return visitArray(rootPath, value.asArrayValue());
    }
    else if (value.isMapValue()) {
        return visitMap(rootPath, value.asMapValue());
    }
    else {
        return value;
    }
}
 
Example 2
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitMap_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: $.json1.k1.k1}",
            "  - {name: $.json1.k2}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{}}", visited.toString());
}
 
Example 3
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitMap_AddColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: $.json1.k3.k3, type: string, default: v}",
            "  - {name: $.json1.k4, src: $.json1.k2}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
 
Example 4
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitMap_addColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
            "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
 
Example 5
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitMap_columnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: \"$['json1']['k1']\"}",
            "  - {name: \"$['json1']['k2']['k2']\"}",
            "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
            "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
 
Example 6
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitArray_dropColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: \"$['json1']['k1'][0]['k1']\"}",
            "  - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
}
 
Example 7
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitArray_addColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
            "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"}],"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
}
 
Example 8
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visitArray_columnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
            "  - {name: \"$['json1']['k2'][0]\"}",
            "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
}
 
Example 9
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visit_withDotAndBracket()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$.json1['k_1']\"}",
            " - {name: \"$.json1['k_1'][0]['k_1']\"}",
            " - {name: \"$['json1']['k_2']\"}",
            " - {name: \"$['json1']['k_2']['k_2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k_1");
    Value k2 = ValueFactory.newString("k_2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
 
Example 10
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visit_withSingleQuotesAndDoubleQuotes()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}",
            " - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}",
            " - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}",
            " - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k.1");
    Value k2 = ValueFactory.newString("k.2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
 
Example 11
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visit_withComplexRename()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$.json1['k____1']\", src: \"$.json1['k.-=+1']\"}",
            " - {name: \"$.json1['k____1'][0]['k____1']\", src: \"$.json1['k____1'][0]['k.-=+1']\"}",
            " - {name: \"$['json1']['k_2']\", src: \"$['json1']['k.2']\"}",
            " - {name: \"$['json1']['k_2']['k_2']\", src: \"$['json1']['k_2']['k.2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k.-=+1");
    Value k2 = ValueFactory.newString("k.2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
 
Example 12
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 6 votes vote down vote up
@Test
public void visit_withColumnNameIncludingSingleQuotes()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$['\\\\'json1']['k1']\"}");
    Schema inputSchema = Schema.builder()
            .add("'json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":"v"}
    Value k1 = ValueFactory.newString("k1");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(k1, v);

    MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
    assertEquals("{\"k1\":\"v\"}", visited.toString());
}
 
Example 13
Source Project: embulk-parser-jsonl   Source File: ColumnCaster.java    License: MIT License 6 votes vote down vote up
public static boolean asBoolean(Value value) throws DataException
{
    if (value.isBooleanValue()) {
        return value.asBooleanValue().getBoolean();
    }
    else if (value.isIntegerValue()) {
        return LongCast.asBoolean(value.asIntegerValue().asLong());
    }
    else if (value.isFloatValue()) {
        return DoubleCast.asBoolean(value.asFloatValue().toDouble());
    }
    else if (value.isStringValue()) {
        return StringCast.asBoolean(value.asStringValue().asString());
    }
    else {
        return JsonCast.asBoolean(value);
    }
}
 
Example 14
Source Project: embulk-parser-jsonl   Source File: ColumnCaster.java    License: MIT License 6 votes vote down vote up
public static long asLong(Value value) throws DataException
{
    if (value.isBooleanValue()) {
        return BooleanCast.asLong(value.asBooleanValue().getBoolean());
    }
    else if (value.isIntegerValue()) {
        return value.asIntegerValue().asLong();
    }
    else if (value.isFloatValue()) {
        return DoubleCast.asLong(value.asFloatValue().toDouble());
    }
    else if (value.isStringValue()) {
        return StringCast.asLong(value.asStringValue().asString());
    }
    else {
        return JsonCast.asLong(value);
    }
}
 
Example 15
Source Project: embulk-parser-jsonl   Source File: ColumnCaster.java    License: MIT License 6 votes vote down vote up
public static double asDouble(Value value) throws DataException
{
    if (value.isBooleanValue()) {
        return BooleanCast.asDouble(value.asBooleanValue().getBoolean());
    }
    else if (value.isIntegerValue()) {
        return LongCast.asDouble(value.asIntegerValue().asLong());
    }
    else if (value.isFloatValue()) {
        return value.asFloatValue().toDouble();
    }
    else if (value.isStringValue()) {
        return StringCast.asDouble(value.asStringValue().asString());
    }
    else {
        return JsonCast.asDouble(value);
    }
}
 
Example 16
Source Project: embulk-parser-jsonl   Source File: ColumnCaster.java    License: MIT License 6 votes vote down vote up
public static Timestamp asTimestamp(Value value, TimestampParser parser) throws DataException
{
    if (value.isBooleanValue()) {
        return BooleanCast.asTimestamp(value.asBooleanValue().getBoolean());
    }
    else if (value.isIntegerValue()) {
        return LongCast.asTimestamp(value.asIntegerValue().asLong());
    }
    else if (value.isFloatValue()) {
        return DoubleCast.asTimestamp(value.asFloatValue().toDouble());
    }
    else if (value.isStringValue()) {
        return StringCast.asTimestamp(value.asStringValue().asString(), parser);
    }
    else {
        return JsonCast.asTimestamp(value);
    }
}
 
Example 17
Source Project: digdag   Source File: TdWaitOperatorFactory.java    License: Apache License 2.0 6 votes vote down vote up
private boolean fetchJobResult(TDJobOperator job)
{
    Optional<ArrayValue> firstRow = pollingRetryExecutor(state, RESULT)
            .retryUnless(TDOperator::isDeterministicClientException)
            .withErrorMessage("Failed to download result of job '%s'", job.getJobId())
            .run(s -> job.getResult(
                    ite -> ite.hasNext()
                            ? Optional.of(ite.next())
                            : Optional.absent()));

    // There must be at least one row in the result for the wait condition to be fulfilled.
    if (!firstRow.isPresent()) {
        return false;
    }

    ArrayValue row = firstRow.get();
    if (row.size() < 1) {
        throw new TaskExecutionException("Got empty row in result of query");
    }

    Value firstCol = row.get(0);
    return isTruthy(firstCol);
}
 
Example 18
Source Project: fluency   Source File: MessagePackRecordAccessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setTimestamp(long timestamp)
{
    int mapSize = mapValueBytes.map().size();
    try (ByteArrayOutputStream output = new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) {
        try (MessagePacker packer = MessagePack.newDefaultPacker(output)) {
            if (mapValueBytes.map().containsKey(KEY_TIME)) {
                packer.packMapHeader(mapSize);
            }
            else {
                packer.packMapHeader(mapSize + 1);
                KEY_TIME.writeTo(packer);
                packer.packLong(timestamp);
            }

            for (Map.Entry<Value, Value> entry : mapValueBytes.map().entrySet()) {
                packer.packValue(entry.getKey());
                packer.packValue(entry.getValue());
            }
        }
        mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray()));
    }
    catch (IOException e) {
        throw new IllegalStateException("Failed to upsert `time` field", e);
    }
}
 
Example 19
Source Project: fluency   Source File: FluentdIngesterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
void ingestWithoutAck()
        throws IOException
{
    Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender);
    ingester.ingest(TAG, ByteBuffer.wrap(DATA));

    verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture());
    List<ByteBuffer> byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0);
    byte[] ingested = getIngestedData(byteBuffers);

    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested);
    ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue();
    assertEquals(3, arrayValue.size());
    assertEquals(TAG, arrayValue.get(0).asStringValue().asString());
    assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray());
    Map<Value, Value> options = arrayValue.get(2).asMapValue().map();
    assertEquals(1, options.size());
    assertEquals(DATA.length,
            options.get(ValueFactory.newString("size")).asIntegerValue().asInt());
}
 
Example 20
Source Project: fluency   Source File: FluencyTest.java    License: Apache License 2.0 6 votes vote down vote up
@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithoutAckResponse(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(1, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertNull(exception);
}
 
Example 21
Source Project: tinkergraph-gremlin   Source File: Deserializer.java    License: Apache License 2.0 5 votes vote down vote up
private Object unpackProperty(final ArrayValue packedValueAndType) {
  final Iterator<Value> iter = packedValueAndType.iterator();
  final byte valueTypeId = iter.next().asIntegerValue().asByte();
  final Value value = iter.next();

  switch (ValueTypes.lookup(valueTypeId)) {
    case BOOLEAN:
      return value.asBooleanValue().getBoolean();
    case STRING:
      return value.asStringValue().asString();
    case BYTE:
      return value.asIntegerValue().asByte();
    case SHORT:
      return value.asIntegerValue().asShort();
    case INTEGER:
      return value.asIntegerValue().asInt();
    case LONG:
      return value.asIntegerValue().asLong();
    case FLOAT:
      return value.asFloatValue().toFloat();
    case DOUBLE:
      return Double.valueOf(value.asFloatValue().toFloat());
    case LIST:
      final ArrayValue arrayValue = value.asArrayValue();
      List deserializedArray = new ArrayList(arrayValue.size());
      final Iterator<Value> valueIterator = arrayValue.iterator();
      while (valueIterator.hasNext()) {
        deserializedArray.add(unpackProperty(valueIterator.next().asArrayValue()));
      }
      return deserializedArray;
    default:
      throw new NotImplementedException("unknown valueTypeId=`" + valueTypeId);
  }
}
 
Example 22
Source Project: embulk-filter-column   Source File: JsonVisitor.java    License: MIT License 5 votes vote down vote up
static Value getDefault(PluginTask task, String name, Type type, ColumnConfig columnConfig)
{
    Object defaultValue = ColumnVisitorImpl.getDefault(task, name, type, columnConfig);
    if (defaultValue == null) {
        return ValueFactory.newNil();
    }
    if (type instanceof BooleanType) {
        return ValueFactory.newBoolean((Boolean) defaultValue);
    }
    else if (type instanceof LongType) {
        return ValueFactory.newInteger((Long) defaultValue);
    }
    else if (type instanceof DoubleType) {
        return ValueFactory.newFloat((Double) defaultValue);
    }
    else if (type instanceof StringType) {
        return ValueFactory.newString((String) defaultValue.toString());
    }
    else if (type instanceof JsonType) {
        return (Value) defaultValue;
    }
    else if (type instanceof TimestampType) {
        throw new ConfigException("type: timestamp is not available in json path");
    }
    else {
        throw new ConfigException(String.format("type: '%s' is not supported", type));
    }
}
 
Example 23
Source Project: embulk-filter-column   Source File: JsonVisitor.java    License: MIT License 5 votes vote down vote up
private void buildJsonAddColumns()
{
    List<ColumnConfig> addColumns = task.getAddColumns();
    for (ColumnConfig column : addColumns) {
        String name = column.getName();
        // skip NON json path notation to build output schema
        if (! PathCompiler.isProbablyJsonPath(name)) {
            continue;
        }
        JsonPathUtil.assertDoNotEndsWithArrayWildcard(name);
        // automatically fill ancestor jsonpaths
        for (JsonColumn ancestorJsonColumn : getAncestorJsonColumnList(name)) {
            String ancestorJsonPath = ancestorJsonColumn.getPath();
            if (!jsonAddColumnsContainsKey(ancestorJsonPath)) {
                jsonAddColumnsPut(ancestorJsonPath, ancestorJsonColumn);
            }
        }
        // leaf jsonpath
        if (column.getSrc().isPresent()) {
            String src = column.getSrc().get();
            jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
        }
        else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
            Type type = column.getType().get();
            Value defaultValue = getDefault(task, name, type, column);
            jsonAddColumnsPut(name, new JsonColumn(name, type, defaultValue));
        }
        else {
            throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"src\", or \"type\" and \"default\"", name));
        }
    }
}
 
Example 24
Source Project: embulk-filter-column   Source File: JsonColumn.java    License: MIT License 5 votes vote down vote up
public JsonColumn(String path, Type type, Value defaultValue, String src)
{
    Path compiledPath = PathCompiler.compile(path);
    Path compiledSrc = src == null ? compiledPath : PathCompiler.compile(src);
    RootPathToken compiledRoot = (RootPathToken) compiledPath.getRoot();
    RootPathToken compiledSrcRoot = (RootPathToken) compiledSrc.getRoot();
    this.path = compiledPath.toString();
    this.type = type;
    this.defaultValue = (defaultValue == null ? ValueFactory.newNil() : defaultValue);
    this.src = compiledSrc.toString();

    this.pathValue = ValueFactory.newString(path);
    this.parentPath = compiledPath.getParentPath();

    this.tailIndex = getTailIndex(compiledRoot);
    this.parentPathValue = ValueFactory.newString(parentPath);
    String tailName = getTailName(compiledRoot);
    this.tailNameValue = tailName == null ? ValueFactory.newNil() : ValueFactory.newString(tailName);

    this.srcValue = ValueFactory.newString(this.src);
    this.srcParentPath = compiledSrc.getParentPath();
    this.srcTailIndex = getTailIndex(compiledSrcRoot);
    this.srcParentPathValue = ValueFactory.newString(this.srcParentPath);
    String srcTailName = getTailName(compiledSrcRoot);
    this.srcTailNameValue = srcTailName == null ? ValueFactory.newNil() : ValueFactory.newString(srcTailName);

    if (!srcParentPath.equals(parentPath)) {
        throw new ConfigException(String.format("The branch (parent path) of src \"%s\" must be same with of name \"%s\" yet", src, path));
    }
}
 
Example 25
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 5 votes vote down vote up
@Test
public void visitMap_Columns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: $.json1.k1}",
            "  - {name: $.json1.k2.k2}",
            "  - {name: $.json1.k3.k3, type: string, default: v}",
            "  - {name: $.json1.k4, src: $.json1.k2}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
 
Example 26
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 5 votes vote down vote up
@Test
public void visitArray_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: \"$.json1.k1[0].k1\"}",
            "  - {name: \"$.json1.k2[*]\"}", // ending with [*] is allowed for drop_columns, but not for others
            "  - {name: \"$.json1.k3[*].k1\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k3":"v"}]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value k3 = ValueFactory.newString("k3");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newArray(v, v),
            k3, ValueFactory.newArray(ValueFactory.newMap(k1, v)));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{}],\"k2\":[],\"k3\":[{}]}", visited.toString());
}
 
Example 27
Source Project: embulk-filter-column   Source File: TestJsonVisitor.java    License: MIT License 5 votes vote down vote up
@Test
public void visitArray_Columns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
            "  - {name: \"$.json1.k2[0]\"}",
            "  - {name: \"$.json1.k3[*].k1\"}",
            "  - {name: \"$.json1.k3[*].k3\", src: \"$.json1.k3[*].k1\"}",
            "  - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
            "  - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"},"v"],"k2":["v","v"],"k3":[{"k1":"v","k2":"v"}]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value k3 = ValueFactory.newString("k3");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
            k2, ValueFactory.newArray(v, v),
            k3, ValueFactory.newArray(ValueFactory.newMap(k1, v, k2, v)));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k1\":\"v\",\"k3\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
}
 
Example 28
Source Project: protostuff   Source File: MsgpackGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private void appendString(StringBuilder sb, Value value)
{
    if (value.isRawValue())
    {
        sb.append(value.toJson());
    }
    else
    {
        sb.append(value.toString());
    }
}
 
Example 29
Source Project: embulk-parser-jsonl   Source File: JsonlParserPlugin.java    License: MIT License 5 votes vote down vote up
private void setColumnNameValues(Schema schema)
{
    ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
    for (Column column : schema.getColumns()) {
        String name = column.getName();
        builder.put(name, newString(name));
    }
    columnNameValues = builder.build();
}
 
Example 30
Source Project: embulk-parser-jsonl   Source File: TestJsonCast.java    License: MIT License 5 votes vote down vote up
@Before
public void createResource()
{
    Value[] kvs = new Value[2];
    kvs[0] = ValueFactory.newString("k");
    kvs[1] = ValueFactory.newString("v");
    value = ValueFactory.newMap(kvs);
}