com.facebook.presto.spi.type.BigintType Java Examples

The following examples show how to use com.facebook.presto.spi.type.BigintType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TestMinimalFunctionality.java From presto-kinesis with Apache License 2.0

6 votes

@Test
public void testStreamHasData()
        throws Exception
{
    MaterializedResult result = queryRunner.execute("Select count(1) from " + streamName);

    MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(0)
            .build();

    assertEquals(result, expected);

    int count = 500;
    createMessages(streamName, count);

    result = queryRunner.execute("SELECT count(1) from " + streamName);

    expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(count)
            .build();

    assertEquals(result, expected);
}

Example #2

Source File: TestJsonDecoder.java From presto-kinesis with Apache License 2.0

6 votes

@Test
public void testStringNumber()
        throws Exception
{
    byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "481516");
    DecoderTestUtil.checkValue(providers, row2, 481516);
    DecoderTestUtil.checkValue(providers, row3, "2342");
    DecoderTestUtil.checkValue(providers, row4, 2342);
}

Example #3

Source File: TestJsonDecoder.java From presto-kinesis with Apache License 2.0

6 votes

@Test
public void testNonExistent()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
}

Example #4

Source File: KubeResTable.java From kubesql with Apache License 2.0

6 votes

public void handleQuantity(Map<String, Quantity>  quantityMap, String prefix, Map<String, Object> data) {
    if (quantityMap != null){
        for (String key : quantityMap.keySet()) {
            String columnName = (prefix + key).toLowerCase();
            if (key.endsWith("cpu")) {
                data.put(columnName, quantityMap.get(key).getNumber().doubleValue());
                if (!getKubeColumn().containsKey(columnName)) {
                    UpdateColumns(columnName, new KubeColumn<Object>(columnName, DoubleType.DOUBLE, key));
                }
            } else {
                data.put(columnName, quantityMap.get(key).getNumber().longValue());
                if (!getKubeColumn().containsKey(columnName)) {
                    UpdateColumns(columnName, new KubeColumn<Object>(columnName, BigintType.BIGINT, key));
                }
            }

        }
    }
}

Example #5

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

6 votes

public static long getLong(Type type, RowResult row, int field) {
    if (type == TimestampType.TIMESTAMP) {
        return row.getLong(field) / 1000;
    } else if (type == BigintType.BIGINT) {
        return row.getLong(field);
    } else if (type == IntegerType.INTEGER) {
        return row.getInt(field);
    } else if (type == SmallintType.SMALLINT) {
        return row.getShort(field);
    } else if (type == TinyintType.TINYINT) {
        return row.getByte(field);
    } else if (type == RealType.REAL) {
        return floatToRawIntBits(row.getFloat(field));
    } else if (type instanceof DecimalType) {
        DecimalType dtype = (DecimalType) type;
        if (dtype.isShort()) {
            return row.getDecimal(field).unscaledValue().longValue();
        } else {
            throw new IllegalStateException("getLong not supported for long decimal: " + type);
        }
    } else {
        throw new IllegalStateException("getLong not implemented for " + type);
    }
}

Example #6

Source File: TestRecordAccess.java From presto-kinesis with Apache License 2.0

6 votes

@Test
public void testStreamHasData()
        throws Exception
{
    MaterializedResult result = queryRunner.execute("Select count(1) from " + dummyStreamName);
    MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(0)
            .build();

    assertEquals(result.getRowCount(), expected.getRowCount());

    int count = 500;
    createDummyMessages(dummyStreamName, count);

    result = queryRunner.execute("SELECT count(1) from " + dummyStreamName);

    expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT)
            .row(count)
            .build();

    assertEquals(result.getRowCount(), expected.getRowCount());
    log.info("Completed second test (select counts)");
}

Example #7

Source File: TestJsonDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testSimple()
        throws Exception
{
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "<a href=\"http://twitterfeed.com\" rel=\"nofollow\">twitterfeed</a>");
    DecoderTestUtil.checkValue(providers, row2, "EKentuckyNews");
    DecoderTestUtil.checkValue(providers, row3, 493857959588286460L);
    DecoderTestUtil.checkValue(providers, row4, 7630);
    DecoderTestUtil.checkValue(providers, row5, true);
}

Example #8

Source File: TestCsvDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testSimple()
{
    String csv = "\"row 1\",row2,\"row3\",100,\"200\",300,4.5";

    CsvKinesisRowDecoder rowDecoder = new CsvKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "1", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "2", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "3", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BigintType.BIGINT, "4", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", BigintType.BIGINT, "5", null, null, false, false);
    KinesisColumnHandle row7 = new KinesisColumnHandle("", 6, "row7", DoubleType.DOUBLE, "6", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6, row7);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "row 1");
    DecoderTestUtil.checkValue(providers, row2, "row2");
    DecoderTestUtil.checkValue(providers, row3, "row3");
    DecoderTestUtil.checkValue(providers, row4, 100);
    DecoderTestUtil.checkValue(providers, row5, 200);
    DecoderTestUtil.checkValue(providers, row6, 300);
    DecoderTestUtil.checkValue(providers, row7, 4.5d);
}

Example #9

Source File: TestRawDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testSimple()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.putLong(4815162342L); // 0 - 7
    buf.putInt(12345678); // 8 - 11
    buf.putShort((short) 4567); // 12 - 13
    buf.put((byte) 123); // 14
    buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, 4815162342L);
    DecoderTestUtil.checkValue(providers, row2, 12345678);
    DecoderTestUtil.checkValue(providers, row3, 4567);
    DecoderTestUtil.checkValue(providers, row4, 123);
    DecoderTestUtil.checkValue(providers, row5, "Ich bin zwei Oeltanks");
}

Example #10

Source File: TestRFC2822JsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkIsNull(providers, row1);
    checkIsNull(providers, row2);
    checkIsNull(providers, row3);
    checkIsNull(providers, row4);
    checkIsNull(providers, row5);
    checkIsNull(providers, row6);
}

Example #11

Source File: TestISO8601JsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}

Example #12

Source File: TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}

Example #13

Source File: TestSecondsSinceEpochJsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testNullValues()
        throws Exception
{
    byte[] json = "{}".getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkIsNull(providers, row1);
    DecoderTestUtil.checkIsNull(providers, row2);
    DecoderTestUtil.checkIsNull(providers, row3);
    DecoderTestUtil.checkIsNull(providers, row4);
    DecoderTestUtil.checkIsNull(providers, row5);
    DecoderTestUtil.checkIsNull(providers, row6);
}

Example #14

Source File: TestJsonDecoder.java From presto-kinesis with Apache License 2.0

5 votes

@Test
public void testOtherExtracts()
        throws Exception
{
    // Test other scenarios: deeper dive into object, get JSON constructs as strings, etc.
    byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/event.json"));

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "event_source", VarcharType.VARCHAR, "source", null, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "user", VarcharType.VARCHAR, "user/handle", null, null, false, false);
    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "user_string", VarcharType.VARCHAR, "user", null, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "timestamp", BigintType.BIGINT, "timestamp", null, null, false, false);
    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "browser_name", VarcharType.VARCHAR, "environment/browser/name", null, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "tags_array", VarcharType.VARCHAR, "tags", null, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    log.info("Decoding row from event JSON file");
    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row1, "otherworld");
    DecoderTestUtil.checkValue(providers, row2, "joeblow");
    DecoderTestUtil.checkValue(providers, row3, "{\"email\":\"[email protected]\",\"handle\":\"joeblow\"}");
    KinesisFieldValueProvider provider = DecoderTestUtil.findValueProvider(providers, row6);
    assertNotNull(provider);
    log.info(new String(provider.getSlice().getBytes(), StandardCharsets.UTF_8));

    DecoderTestUtil.checkValue(providers, row4, 1450214872847L);
    DecoderTestUtil.checkValue(providers, row5, "Chrome");
    DecoderTestUtil.checkValue(providers, row6, "[\"tag1\",\"tag2\",\"tag3\"]");
    log.info("DONE");
}

Example #15

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

5 votes

public static Object getObject(Type type, RowResult row, int field) {
    if (row.isNull(field)) {
        return null;
    } else {
        if (type instanceof VarcharType) {
            return row.getString(field);
        } else if (type == TimestampType.TIMESTAMP) {
            return row.getLong(field) / 1000;
        } else if (type == BigintType.BIGINT) {
            return row.getLong(field);
        } else if (type == IntegerType.INTEGER) {
            return row.getInt(field);
        } else if (type == SmallintType.SMALLINT) {
            return row.getShort(field);
        } else if (type == TinyintType.TINYINT) {
            return row.getByte(field);
        } else if (type == DoubleType.DOUBLE) {
            return row.getDouble(field);
        } else if (type == RealType.REAL) {
            return row.getFloat(field);
        } else if (type == BooleanType.BOOLEAN) {
            return row.getBoolean(field);
        } else if (type instanceof VarbinaryType) {
            return Slices.wrappedBuffer(row.getBinary(field));
        } else if (type instanceof DecimalType) {
            return row.getDecimal(field);
        } else {
            throw new IllegalStateException("getObject not implemented for " + type);
        }
    }
}

Example #16

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

5 votes

public static Object getJavaValue(Type type, Object nativeValue) {
    if (type instanceof VarcharType) {
        return ((Slice) nativeValue).toStringUtf8();
    } else if (type == TimestampType.TIMESTAMP) {
        return ((Long) nativeValue) * 1000;
    } else if (type == BigintType.BIGINT) {
        return nativeValue;
    } else if (type == IntegerType.INTEGER) {
        return ((Long) nativeValue).intValue();
    } else if (type == SmallintType.SMALLINT) {
        return ((Long) nativeValue).shortValue();
    } else if (type == TinyintType.TINYINT) {
        return ((Long) nativeValue).byteValue();
    } else if (type == DoubleType.DOUBLE) {
        return nativeValue;
    } else if (type == RealType.REAL) {
        // conversion can result in precision lost
        return intBitsToFloat(((Long) nativeValue).intValue());
    } else if (type == BooleanType.BOOLEAN) {
        return nativeValue;
    } else if (type instanceof VarbinaryType) {
        return ((Slice) nativeValue).toByteBuffer();
    } else if (type instanceof DecimalType) {
        return nativeValue;
    } else {
        throw new IllegalStateException("Back conversion not implemented for " + type);
    }
}

Example #17

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

5 votes

public static NullableValue getColumnValue(Type type, PartialRow row, int i) {
    if (row.isNull(i)) {
        return NullableValue.asNull(type);
    } else {
        if (type instanceof VarcharType) {
            return NullableValue.of(type, utf8Slice(row.getString(i)));
        } else if (type == TimestampType.TIMESTAMP) {
            return NullableValue.of(type, row.getLong(i) / 1000);
        } else if (type == BigintType.BIGINT) {
            return NullableValue.of(type, row.getLong(i));
        } else if (type == IntegerType.INTEGER) {
            return NullableValue.of(type, row.getInt(i));
        } else if (type == SmallintType.SMALLINT) {
            return NullableValue.of(type, row.getShort(i));
        } else if (type == TinyintType.TINYINT) {
            return NullableValue.of(type, row.getByte(i));
        } else if (type == DoubleType.DOUBLE) {
            return NullableValue.of(type, row.getDouble(i));
        } else if (type == RealType.REAL) {
            return NullableValue.of(type, (long) floatToRawIntBits(row.getFloat(i)));
        } else if (type == BooleanType.BOOLEAN) {
            return NullableValue.of(type, row.getBoolean(i));
        } else if (type instanceof VarbinaryType) {
            return NullableValue.of(type, wrappedBuffer(row.getBinary(i)));
        } else if (type instanceof DecimalType) {
            return NullableValue.of(type, row.getDecimal(i));
        } else {
            throw new IllegalStateException("Handling of type " + type + " is not implemented");
        }
    }
}

Example #18

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

5 votes

private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes) {
    switch (ktype) {
        case STRING:
            return VarcharType.VARCHAR;
        case UNIXTIME_MICROS:
            return TimestampType.TIMESTAMP;
        case INT64:
            return BigintType.BIGINT;
        case INT32:
            return IntegerType.INTEGER;
        case INT16:
            return SmallintType.SMALLINT;
        case INT8:
            return TinyintType.TINYINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BOOL:
            return BooleanType.BOOLEAN;
        case BINARY:
            return VarbinaryType.VARBINARY;
        case DECIMAL:
            return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale());
        default:
            throw new IllegalStateException("Kudu type not implemented for " + ktype);
    }
}

Example #19

Source File: TestUtils.java From presto-kinesis with Apache License 2.0

5 votes

public static Map.Entry<SchemaTableName, KinesisStreamDescription> createSimpleJsonStreamDescription(String streamName, SchemaTableName schemaTableName)
{
    // Format: {"id" : 1324, "name" : "some string"}
    ArrayList<KinesisStreamFieldDescription> fieldList = new ArrayList<KinesisStreamFieldDescription>();
    fieldList.add(new KinesisStreamFieldDescription("id", BigintType.BIGINT, "id", "comment", null, null, false));
    fieldList.add(new KinesisStreamFieldDescription("name", VarcharType.VARCHAR, "name", "comment", null, null, false));
    KinesisStreamFieldGroup grp = new KinesisStreamFieldGroup("json", fieldList);

    KinesisStreamDescription desc = new KinesisStreamDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), streamName, grp);
    return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, desc);
}

Example #20

Source File: TypeHelper.java From presto-kudu with Apache License 2.0

5 votes

public static org.apache.kudu.Type toKuduClientType(Type type) {
    if (type instanceof VarcharType) {
        return org.apache.kudu.Type.STRING;
    } else if (type == TimestampType.TIMESTAMP) {
        return org.apache.kudu.Type.UNIXTIME_MICROS;
    } else if (type == BigintType.BIGINT) {
        return org.apache.kudu.Type.INT64;
    } else if (type == IntegerType.INTEGER) {
        return org.apache.kudu.Type.INT32;
    } else if (type == SmallintType.SMALLINT) {
        return org.apache.kudu.Type.INT16;
    } else if (type == TinyintType.TINYINT) {
        return org.apache.kudu.Type.INT8;
    } else if (type == RealType.REAL) {
        return org.apache.kudu.Type.FLOAT;
    } else if (type == DoubleType.DOUBLE) {
        return org.apache.kudu.Type.DOUBLE;
    } else if (type == BooleanType.BOOLEAN) {
        return org.apache.kudu.Type.BOOL;
    } else if (type instanceof VarbinaryType) {
        return org.apache.kudu.Type.BINARY;
    } else if (type instanceof DecimalType) {
        return org.apache.kudu.Type.DECIMAL;
    } else if (type == DateType.DATE) {
        return org.apache.kudu.Type.STRING;
    } else if (type instanceof CharType) {
        return org.apache.kudu.Type.STRING;
    } else {
        throw new IllegalStateException("Type mapping implemented for Presto type: " + type);
    }
}

Example #21

Source File: Elasticsearch2Client.java From presto-connectors with Apache License 2.0

4 votes

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //es 2.4.x
        // .field("index", "not_analyzed")
        return fieldBuilder.field("type", "string");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}

Example #22

Source File: Elasticsearch6Client.java From presto-connectors with Apache License 2.0

4 votes

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //TODO: text or keyword ?
        return fieldBuilder.field("type", "text");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}

Example #23

Source File: TestRawDecoder.java From presto-kinesis with Apache License 2.0

4 votes

@Test
public void testBooleanStuff()
{
    ByteBuffer buf = ByteBuffer.allocate(100);
    buf.put((byte) 127); // offset 0
    buf.putLong(0); // offset 1
    buf.put((byte) 126); // offset 9
    buf.putLong(1); // offset 10

    buf.put((byte) 125); // offset 18
    buf.putInt(0); // offset 19
    buf.put((byte) 124); // offset 23
    buf.putInt(1); // offset 24

    buf.put((byte) 123); // offset 28
    buf.putShort((short) 0); // offset 29
    buf.put((byte) 122); // offset 31
    buf.putShort((short) 1); // offset 32

    buf.put((byte) 121); // offset 34
    buf.put((byte) 0); // offset 35
    buf.put((byte) 120); // offset 36
    buf.put((byte) 1); // offset 37

    byte[] row = new byte[buf.position()];
    System.arraycopy(buf.array(), 0, row, 0, buf.position());

    RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder();
    KinesisColumnHandle row01 = new KinesisColumnHandle("", 0, "row01", BigintType.BIGINT, "0", "BYTE", null, false, false);
    KinesisColumnHandle row02 = new KinesisColumnHandle("", 1, "row02", BooleanType.BOOLEAN, "1", "LONG", null, false, false);
    KinesisColumnHandle row03 = new KinesisColumnHandle("", 2, "row03", BigintType.BIGINT, "9", "BYTE", null, false, false);
    KinesisColumnHandle row04 = new KinesisColumnHandle("", 3, "row04", BooleanType.BOOLEAN, "10", "LONG", null, false, false);

    KinesisColumnHandle row11 = new KinesisColumnHandle("", 4, "row11", BigintType.BIGINT, "18", "BYTE", null, false, false);
    KinesisColumnHandle row12 = new KinesisColumnHandle("", 5, "row12", BooleanType.BOOLEAN, "19", "INT", null, false, false);
    KinesisColumnHandle row13 = new KinesisColumnHandle("", 6, "row13", BigintType.BIGINT, "23", "BYTE", null, false, false);
    KinesisColumnHandle row14 = new KinesisColumnHandle("", 7, "row14", BooleanType.BOOLEAN, "24", "INT", null, false, false);

    KinesisColumnHandle row21 = new KinesisColumnHandle("", 8, "row21", BigintType.BIGINT, "28", "BYTE", null, false, false);
    KinesisColumnHandle row22 = new KinesisColumnHandle("", 9, "row22", BooleanType.BOOLEAN, "29", "SHORT", null, false, false);
    KinesisColumnHandle row23 = new KinesisColumnHandle("", 10, "row23", BigintType.BIGINT, "31", "BYTE", null, false, false);
    KinesisColumnHandle row24 = new KinesisColumnHandle("", 11, "row24", BooleanType.BOOLEAN, "32", "SHORT", null, false, false);

    KinesisColumnHandle row31 = new KinesisColumnHandle("", 12, "row31", BigintType.BIGINT, "34", "BYTE", null, false, false);
    KinesisColumnHandle row32 = new KinesisColumnHandle("", 13, "row32", BooleanType.BOOLEAN, "35", "BYTE", null, false, false);
    KinesisColumnHandle row33 = new KinesisColumnHandle("", 14, "row33", BigintType.BIGINT, "36", "BYTE", null, false, false);
    KinesisColumnHandle row34 = new KinesisColumnHandle("", 15, "row34", BooleanType.BOOLEAN, "37", "BYTE", null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row01,
            row02,
            row03,
            row04,
            row11,
            row12,
            row13,
            row14,
            row21,
            row22,
            row23,
            row24,
            row31,
            row32,
            row33,
            row34);

    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    DecoderTestUtil.checkValue(providers, row01, 127);
    DecoderTestUtil.checkValue(providers, row02, false);
    DecoderTestUtil.checkValue(providers, row03, 126);
    DecoderTestUtil.checkValue(providers, row04, true);

    DecoderTestUtil.checkValue(providers, row11, 125);
    DecoderTestUtil.checkValue(providers, row12, false);
    DecoderTestUtil.checkValue(providers, row13, 124);
    DecoderTestUtil.checkValue(providers, row14, true);

    DecoderTestUtil.checkValue(providers, row21, 123);
    DecoderTestUtil.checkValue(providers, row22, false);
    DecoderTestUtil.checkValue(providers, row23, 122);
    DecoderTestUtil.checkValue(providers, row24, true);

    DecoderTestUtil.checkValue(providers, row31, 121);
    DecoderTestUtil.checkValue(providers, row32, false);
    DecoderTestUtil.checkValue(providers, row33, 120);
    DecoderTestUtil.checkValue(providers, row34, true);
}

Example #24

Source File: Elasticsearch5Client.java From presto-connectors with Apache License 2.0

4 votes

private static XContentBuilder buildFieldType(XContentBuilder fieldBuilder, Type type)
        throws IOException
{
    final String dateTimeFormat = "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis";
    //Type mapping
    // see:https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping-types.html
    if (type.equals(BooleanType.BOOLEAN)) {
        return fieldBuilder.field("type", "boolean");
    }
    if (type.equals(BigintType.BIGINT)) {
        return fieldBuilder.field("type", "long");
    }
    if (type.equals(IntegerType.INTEGER)) {
        return fieldBuilder.field("type", "integer");
    }
    if (type.equals(SmallintType.SMALLINT)) {
        return fieldBuilder.field("type", "short");
    }
    if (type.equals(TinyintType.TINYINT)) {
        return fieldBuilder.field("type", "byte");
    }
    if (type.equals(DoubleType.DOUBLE)) {
        return fieldBuilder.field("type", "double");
    }
    if (type.equals(DateType.DATE)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimeType.TIME)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampType.TIMESTAMP)) {
        return fieldBuilder.field("type", "date")
                .field("format", dateTimeFormat);
    }
    if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) {
        //TODO: TIMESTAMP_WITH_TIME_ZONE
        return fieldBuilder.field("type", "date")
                .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis");
    }
    if (type instanceof DecimalType) {
        return fieldBuilder.field("type", "double");
    }
    if (isVarcharType(type)) {
        //TODO: text or keyword ?
        return fieldBuilder.field("type", "text");
    }
    if (type.equals(VarbinaryType.VARBINARY)) {
        return fieldBuilder.field("type", "binary");
    }
    if (isArrayType(type)) {
        Type elementType = type.getTypeParameters().get(0);
        if (isArrayType(elementType) || isMapType(elementType) || isRowType(elementType)) {
            throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
        }
        return buildFieldType(fieldBuilder, elementType);
    }
    if (isMapType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }
    if (isRowType(type)) {
        throw new PrestoException(NOT_SUPPORTED, "sorry unsupported type: " + type);
    }

    throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type);
}

Example #25

Source File: ParaflowMetaDataReader.java From paraflow with Apache License 2.0

4 votes

private Type getType(String typeName)
{
    log.debug("Get type " + typeName);
    typeName = typeName.toLowerCase();
    // check if type is varchar(xx)
    Pattern vcpattern = Pattern.compile("varchar\\(\\s*(\\d+)\\s*\\)");
    Matcher vcmatcher = vcpattern.matcher(typeName);
    if (vcmatcher.find()) {
        String vlen = vcmatcher.group(1);
        if (!vlen.isEmpty()) {
            return VarcharType.createVarcharType(Integer.parseInt(vlen));
        }
        return UnknownType.UNKNOWN;
    }
    // check if type is char(xx)
    Pattern cpattern = Pattern.compile("char\\(\\s*(\\d+)\\s*\\)");
    Matcher cmatcher = cpattern.matcher(typeName);
    if (cmatcher.find()) {
        String clen = cmatcher.group(1);
        if (!clen.isEmpty()) {
            return CharType.createCharType(Integer.parseInt(clen));
        }
        return UnknownType.UNKNOWN;
    }
    // check if type is decimal(precision, scale)
    Pattern dpattern = Pattern.compile("decimal\\((\\d+)\\s*,?\\s*(\\d*)\\)");
    Matcher dmatcher = dpattern.matcher(typeName);
    if (dmatcher.find()) {
        String dprecision = dmatcher.group(1);
        String dscale = dmatcher.group(2);
        if (dprecision.isEmpty()) {
            return UnknownType.UNKNOWN;
        }
        if (dscale.isEmpty()) {
            return DecimalType.createDecimalType(Integer.parseInt(dprecision));
        }
        return DecimalType.createDecimalType(Integer.parseInt(dprecision), Integer.parseInt(dscale));
    }
    switch (typeName) {
        case "boolean":
            return BooleanType.BOOLEAN;
        case "tinyint":
            return TinyintType.TINYINT;
        case "smallint":
            return SmallintType.SMALLINT;
        case "integer":
            return IntegerType.INTEGER;
        case "bigint":
            return BigintType.BIGINT;
        case "real":
            return RealType.REAL;
        case "double":
            return DoubleType.DOUBLE;
        case "date":
            return DateType.DATE;
        case "time":
            return TimeType.TIME;
        case "timestamp":
            return TimestampType.TIMESTAMP;
        default:
            return UnknownType.UNKNOWN;
    }
}

Example #26

Source File: TestRFC2822JsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

4 votes

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = (System.currentTimeMillis() / 1000) * 1000; // rfc2822 is second granularity
    String nowString = FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    checkValue(providers, row1, now);
    checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    checkValue(providers, row6, nowString);
}

Example #27

Source File: EthereumMetadata.java From presto-ethereum with Apache License 2.0

4 votes

@SuppressWarnings("ValueOfIncrementOrDecrementUsed")
@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) {
    EthereumTableHandle ethereumTableHandle = convertTableHandle(tableHandle);
    String tableName = ethereumTableHandle.getTableName();

    ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
    int index = 0;
    if (EthereumTable.BLOCK.getName().equals(tableName)) {
        columnHandles.put("block_number", new EthereumColumnHandle(connectorId, index++, "block_number", BigintType.BIGINT));
        columnHandles.put("block_hash", new EthereumColumnHandle(connectorId, index++, "block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_parentHash", new EthereumColumnHandle(connectorId, index++, "block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_nonce", new EthereumColumnHandle(connectorId, index++, "block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_sha3Uncles", new EthereumColumnHandle(connectorId, index++, "block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_logsBloom", new EthereumColumnHandle(connectorId, index++, "block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_transactionsRoot", new EthereumColumnHandle(connectorId, index++, "block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_stateRoot", new EthereumColumnHandle(connectorId, index++, "block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_miner", new EthereumColumnHandle(connectorId, index++, "block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("block_difficulty", new EthereumColumnHandle(connectorId, index++, "block_difficulty", BigintType.BIGINT));
        columnHandles.put("block_totalDifficulty", new EthereumColumnHandle(connectorId, index++, "block_totalDifficulty", BigintType.BIGINT));
        columnHandles.put("block_size", new EthereumColumnHandle(connectorId, index++, "block_size", IntegerType.INTEGER));
        columnHandles.put("block_extraData", new EthereumColumnHandle(connectorId, index++, "block_extraData", VarcharType.VARCHAR));
        columnHandles.put("block_gasLimit", new EthereumColumnHandle(connectorId, index++, "block_gasLimit", DoubleType.DOUBLE));
        columnHandles.put("block_gasUsed", new EthereumColumnHandle(connectorId, index++, "block_gasUsed", DoubleType.DOUBLE));
        columnHandles.put("block_timestamp", new EthereumColumnHandle(connectorId, index++, "block_timestamp", BigintType.BIGINT));
        columnHandles.put("block_transactions", new EthereumColumnHandle(connectorId, index++, "block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
        columnHandles.put("block_uncles", new EthereumColumnHandle(connectorId, index++, "block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
    } else if (EthereumTable.TRANSACTION.getName().equals(tableName)) {
        columnHandles.put("tx_hash", new EthereumColumnHandle(connectorId, index++, "tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_nonce", new EthereumColumnHandle(connectorId, index++, "tx_nonce", BigintType.BIGINT));
        columnHandles.put("tx_blockHash", new EthereumColumnHandle(connectorId, index++, "tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_blockNumber", new EthereumColumnHandle(connectorId, index++, "tx_blockNumber", BigintType.BIGINT));
        columnHandles.put("tx_transactionIndex", new EthereumColumnHandle(connectorId, index++, "tx_transactionIndex", IntegerType.INTEGER));
        columnHandles.put("tx_from", new EthereumColumnHandle(connectorId, index++, "tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_to", new EthereumColumnHandle(connectorId, index++, "tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("tx_value", new EthereumColumnHandle(connectorId, index++, "tx_value", DoubleType.DOUBLE));
        columnHandles.put("tx_gas", new EthereumColumnHandle(connectorId, index++, "tx_gas", DoubleType.DOUBLE));
        columnHandles.put("tx_gasPrice", new EthereumColumnHandle(connectorId, index++, "tx_gasPrice", DoubleType.DOUBLE));
        columnHandles.put("tx_input", new EthereumColumnHandle(connectorId, index++, "tx_input", VarcharType.VARCHAR));
    } else if (EthereumTable.ERC20.getName().equals(tableName)) {
        columnHandles.put("erc20_token", new EthereumColumnHandle(connectorId, index++, "erc20_token", VarcharType.createUnboundedVarcharType()));
        columnHandles.put("erc20_from", new EthereumColumnHandle(connectorId, index++, "erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_to", new EthereumColumnHandle(connectorId, index++, "erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_value", new EthereumColumnHandle(connectorId, index++, "erc20_value", DoubleType.DOUBLE));
        columnHandles.put("erc20_txHash", new EthereumColumnHandle(connectorId, index++, "erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        columnHandles.put("erc20_blockNumber", new EthereumColumnHandle(connectorId, index++, "erc20_blockNumber", BigintType.BIGINT));
    } else {
        throw new IllegalArgumentException("Unknown Table Name " + tableName);
    }

    return columnHandles.build();
}

Example #28

Source File: TestISO8601JsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

4 votes

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = PRINTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, nowString);

    // number parsed as number --> as is
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, Long.toString(now));

    // string parsed as string --> as is
    DecoderTestUtil.checkValue(providers, row6, nowString);
}

Example #29

Source File: EthereumMetadata.java From presto-ethereum with Apache License 2.0

4 votes

@SuppressWarnings("ValueOfIncrementOrDecrementUsed")
private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) {
    ImmutableList.Builder<ColumnMetadata> builder = ImmutableList.builder();

    if (EthereumTable.BLOCK.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("block_number", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_parentHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_nonce", VarcharType.createVarcharType(H8_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_sha3Uncles", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_logsBloom", VarcharType.createVarcharType(H256_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_transactionsRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_stateRoot", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_miner", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("block_difficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_totalDifficulty", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_size", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("block_extraData", VarcharType.VARCHAR));
        builder.add(new ColumnMetadata("block_gasLimit", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_gasUsed", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("block_timestamp", BigintType.BIGINT));
        builder.add(new ColumnMetadata("block_transactions", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
        builder.add(new ColumnMetadata("block_uncles", new ArrayType(VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH))));
    } else if (EthereumTable.TRANSACTION.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("tx_hash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_nonce", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_blockHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_blockNumber", BigintType.BIGINT));
        builder.add(new ColumnMetadata("tx_transactionIndex", IntegerType.INTEGER));
        builder.add(new ColumnMetadata("tx_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("tx_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gas", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_gasPrice", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("tx_input", VarcharType.VARCHAR));
    } else if (EthereumTable.ERC20.getName().equals(schemaTableName.getTableName())) {
        builder.add(new ColumnMetadata("erc20_token", VarcharType.createUnboundedVarcharType()));
        builder.add(new ColumnMetadata("erc20_from", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_to", VarcharType.createVarcharType(H20_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_value", DoubleType.DOUBLE));
        builder.add(new ColumnMetadata("erc20_txHash", VarcharType.createVarcharType(H32_BYTE_HASH_STRING_LENGTH)));
        builder.add(new ColumnMetadata("erc20_blockNumber", BigintType.BIGINT));
    } else {
        throw new IllegalArgumentException("Unknown Table Name " + schemaTableName.getTableName());
    }

    return new ConnectorTableMetadata(schemaTableName, builder.build());
}

Example #30

Source File: TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java From presto-kinesis with Apache License 2.0

4 votes

@Test
public void testBasicFormatting()
        throws Exception
{
    long now = System.currentTimeMillis();
    String nowString = MillisecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER.print(now);

    byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8);

    JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get());
    KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);
    KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME, null, false, false);

    KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);
    KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false);

    List<KinesisColumnHandle> columns = ImmutableList.of(row1, row2, row3, row4, row5, row6);
    Set<KinesisFieldValueProvider> providers = new HashSet<>();

    boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns));
    assertTrue(valid);

    assertEquals(providers.size(), columns.size());

    // sanity checks
    DecoderTestUtil.checkValue(providers, row1, now);
    DecoderTestUtil.checkValue(providers, row2, Long.toString(now));

    // number parsed as number --> return as time stamp (millis)
    DecoderTestUtil.checkValue(providers, row3, now);
    // string parsed as number --> parse text, convert to timestamp
    DecoderTestUtil.checkValue(providers, row4, now);

    // number parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row5, nowString);

    // string parsed as string --> parse text, convert to timestamp, turn into string
    DecoderTestUtil.checkValue(providers, row6, nowString);
}