com.amazonaws.services.glue.model.Column Java Examples

The following examples show how to use com.amazonaws.services.glue.model.Column. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestObjects.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
public static List<Column> getTestFieldList() {
  List<Column> fieldList = new ArrayList<>();
  Column field = new Column()
      .withComment(UUID.randomUUID().toString())
      .withName("column" + UUID.randomUUID().toString().replaceAll("[^a-zA-Z0-9]+", ""))
      .withType("string");
  fieldList.add(field);
  return fieldList;
}
Example #2
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up 
@Test
public void testListPartitionNames() throws Exception {
  List<String> values1 = Arrays.asList("a", "x");
  Partition partition1 = new Partition()
      .withDatabaseName(testDB.getName())
      .withTableName(testTable.getTableName())
      .withValues(values1);

  List<String> values2 = Arrays.asList("a", "y");
  Partition partition2 = new Partition()
      .withDatabaseName(testDB.getName())
      .withTableName(testTable.getTableName())
      .withValues(values2);

  Table table = HiveToCatalogConverter.convertTable(testTable);
  table.setPartitionKeys(Arrays.asList(new Column().withName("foo"), new Column().withName("bar")));

  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
      .thenReturn(new GetPartitionsResult().withPartitions(Arrays.asList(partition1, partition2)));

  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(table));

  List<String> partitionNames = metastoreClient.listPartitionNames(
      testDB.getName(), testTable.getTableName(), (short) -1);
  assertThat(partitionNames, containsInAnyOrder("foo=a/bar=x", "foo=a/bar=y"));
}
Example #3
Source File: DynamoDBMetadataHandlerTest.java    From aws-athena-query-federation with Apache License 2.0 5 votes vote down vote up 
@Test
public void validateSourceTableNamePropagation()
        throws Exception
{
    List<Column> columns = new ArrayList<>();
    columns.add(new Column().withName("col1").withType("int"));
    columns.add(new Column().withName("col2").withType("bigint"));
    columns.add(new Column().withName("col3").withType("string"));

    Map<String, String> param = ImmutableMap.of(
            SOURCE_TABLE_PROPERTY, TEST_TABLE,
            COLUMN_NAME_MAPPING_PROPERTY, "col1=Col1 , col2=Col2 ,col3=Col3",
            DATETIME_FORMAT_MAPPING_PROPERTY, "col1=datetime1,col3=datetime3 ");
    Table table = new Table()
            .withParameters(param)
            .withPartitionKeys()
            .withStorageDescriptor(new StorageDescriptor().withColumns(columns));
    GetTableResult mockResult = new GetTableResult().withTable(table);
    when(glueClient.getTable(any())).thenReturn(mockResult);

    TableName tableName = new TableName(DEFAULT_SCHEMA, "glueTableForTestTable");
    GetTableRequest getTableRequest = new GetTableRequest(TEST_IDENTITY, TEST_QUERY_ID, TEST_CATALOG_NAME, tableName);
    GetTableResponse getTableResponse = handler.doGetTable(allocator, getTableRequest);
    logger.info("validateSourceTableNamePropagation: GetTableResponse[{}]", getTableResponse);
    Map<String, String> customMetadata = getTableResponse.getSchema().getCustomMetadata();
    assertThat(customMetadata.get(SOURCE_TABLE_PROPERTY), equalTo(TEST_TABLE));
    assertThat(customMetadata.get(DATETIME_FORMAT_MAPPING_PROPERTY_NORMALIZED), equalTo("Col1=datetime1,Col3=datetime3"));

    GetTableLayoutRequest getTableLayoutRequest = new GetTableLayoutRequest(TEST_IDENTITY,
            TEST_QUERY_ID,
            TEST_CATALOG_NAME,
            tableName,
            new Constraints(ImmutableMap.of()),
            getTableResponse.getSchema(),
            Collections.EMPTY_SET);

    GetTableLayoutResponse getTableLayoutResponse = handler.doGetTableLayout(allocator, getTableLayoutRequest);
    logger.info("validateSourceTableNamePropagation: GetTableLayoutResponse[{}]", getTableLayoutResponse);
    assertThat(getTableLayoutResponse.getPartitions().getSchema().getCustomMetadata().get(TABLE_METADATA), equalTo(TEST_TABLE));
}
Example #4
Source File: TestingMetastoreObjects.java    From presto with Apache License 2.0 5 votes vote down vote up 
public static Column getGlueTestColumn()
{
    return new Column()
            .withName("test-col" + generateRandom())
            .withType("string")
            .withComment("column comment");
}
Example #5
Source File: GlueMetadataHandlerTest.java    From aws-athena-query-federation with Apache License 2.0 4 votes vote down vote up 
@Test
public void doGetTable()
        throws Exception
{
    String sourceTable = "My-Table";

    Map<String, String> expectedParams = new HashMap<>();
    expectedParams.put(SOURCE_TABLE_PROPERTY, sourceTable);
    expectedParams.put(COLUMN_NAME_MAPPING_PROPERTY, "col2=Col2,col3=Col3, col4=Col4");
    expectedParams.put(DATETIME_FORMAT_MAPPING_PROPERTY, "col2=someformat2, col1=someformat1 ");

    List<Column> columns = new ArrayList<>();
    columns.add(new Column().withName("col1").withType("int").withComment("comment"));
    columns.add(new Column().withName("col2").withType("bigint").withComment("comment"));
    columns.add(new Column().withName("col3").withType("string").withComment("comment"));
    columns.add(new Column().withName("col4").withType("timestamp").withComment("comment"));
    columns.add(new Column().withName("col5").withType("date").withComment("comment"));
    columns.add(new Column().withName("col6").withType("timestamptz").withComment("comment"));
    columns.add(new Column().withName("col7").withType("timestamptz").withComment("comment"));

    Table mockTable = mock(Table.class);
    StorageDescriptor mockSd = mock(StorageDescriptor.class);

    when(mockTable.getName()).thenReturn(table);
    when(mockTable.getStorageDescriptor()).thenReturn(mockSd);
    when(mockTable.getParameters()).thenReturn(expectedParams);
    when(mockSd.getColumns()).thenReturn(columns);

    when(mockGlue.getTable(any(com.amazonaws.services.glue.model.GetTableRequest.class)))
            .thenAnswer((InvocationOnMock invocationOnMock) ->
            {
                com.amazonaws.services.glue.model.GetTableRequest request =
                        (com.amazonaws.services.glue.model.GetTableRequest) invocationOnMock.getArguments()[0];

                assertEquals(accountId, request.getCatalogId());
                assertEquals(schema, request.getDatabaseName());
                assertEquals(table, request.getName());

                GetTableResult mockResult = mock(GetTableResult.class);
                when(mockResult.getTable()).thenReturn(mockTable);
                return mockResult;
            });

    GetTableRequest req = new GetTableRequest(IdentityUtil.fakeIdentity(), queryId, catalog, new TableName(schema, table));
    GetTableResponse res = handler.doGetTable(allocator, req);

    logger.info("doGetTable - {}", res);

    assertTrue(res.getSchema().getFields().size() == 7);
    assertTrue(res.getSchema().getCustomMetadata().size() > 0);
    assertTrue(res.getSchema().getCustomMetadata().containsKey(DATETIME_FORMAT_MAPPING_PROPERTY));
    assertEquals(res.getSchema().getCustomMetadata().get(DATETIME_FORMAT_MAPPING_PROPERTY_NORMALIZED), "Col2=someformat2,col1=someformat1");
    assertEquals(sourceTable, getSourceTableName(res.getSchema()));

    //Verify column name mapping works
    assertNotNull(res.getSchema().findField("col1"));
    assertNotNull(res.getSchema().findField("Col2"));
    assertNotNull(res.getSchema().findField("Col3"));
    assertNotNull(res.getSchema().findField("Col4"));
    assertNotNull(res.getSchema().findField("col5"));
    assertNotNull(res.getSchema().findField("col6"));
    assertNotNull(res.getSchema().findField("col7"));

    //Verify types
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("col1").getType()).equals(Types.MinorType.INT));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("Col2").getType()).equals(Types.MinorType.BIGINT));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("Col3").getType()).equals(Types.MinorType.VARCHAR));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("Col4").getType()).equals(Types.MinorType.DATEMILLI));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("col5").getType()).equals(Types.MinorType.DATEDAY));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("col6").getType()).equals(Types.MinorType.TIMESTAMPMILLITZ));
    assertTrue(Types.getMinorTypeForArrowType(res.getSchema().findField("col7").getType()).equals(Types.MinorType.TIMESTAMPMILLITZ));
}
Example #6
Source File: DynamoDBMetadataHandlerTest.java    From aws-athena-query-federation with Apache License 2.0 4 votes vote down vote up 
@Test
public void doGetTableLayoutScanWithTypeOverride()
        throws Exception
{
    List<Column> columns = new ArrayList<>();
    columns.add(new Column().withName("col1").withType("int"));
    columns.add(new Column().withName("col2").withType("timestamptz"));
    columns.add(new Column().withName("col3").withType("string"));

    Map<String, String> param = ImmutableMap.of(
            SOURCE_TABLE_PROPERTY, TEST_TABLE,
            COLUMN_NAME_MAPPING_PROPERTY, "col1=Col1",
            DATETIME_FORMAT_MAPPING_PROPERTY, "col1=datetime1,col3=datetime3 ");
    Table table = new Table()
            .withParameters(param)
            .withPartitionKeys()
            .withStorageDescriptor(new StorageDescriptor().withColumns(columns));
    GetTableResult mockResult = new GetTableResult().withTable(table);
    when(glueClient.getTable(any())).thenReturn(mockResult);

    TableName tableName = new TableName(DEFAULT_SCHEMA, "glueTableForTestTable");
    GetTableRequest getTableRequest = new GetTableRequest(TEST_IDENTITY, TEST_QUERY_ID, TEST_CATALOG_NAME, tableName);
    GetTableResponse getTableResponse = handler.doGetTable(allocator, getTableRequest);
    logger.info("validateSourceTableNamePropagation: GetTableResponse[{}]", getTableResponse);
    Map<String, String> customMetadata = getTableResponse.getSchema().getCustomMetadata();
    assertThat(customMetadata.get(SOURCE_TABLE_PROPERTY), equalTo(TEST_TABLE));
    assertThat(customMetadata.get(DATETIME_FORMAT_MAPPING_PROPERTY_NORMALIZED), equalTo("Col1=datetime1,col3=datetime3"));

    Map<String, ValueSet> constraintsMap = new HashMap<>();
    constraintsMap.put("col3",
            EquatableValueSet.newBuilder(allocator, new ArrowType.Bool(), true, true)
                    .add(true).build());
    constraintsMap.put("col2",
            EquatableValueSet.newBuilder(allocator, new ArrowType.Bool(), true, true)
                    .add(true).build());

    GetTableLayoutRequest getTableLayoutRequest = new GetTableLayoutRequest(TEST_IDENTITY,
            TEST_QUERY_ID,
            TEST_CATALOG_NAME,
            tableName,
            new Constraints(constraintsMap),
            getTableResponse.getSchema(),
            Collections.EMPTY_SET);


    GetTableLayoutResponse res = handler.doGetTableLayout(allocator, getTableLayoutRequest);

    logger.info("doGetTableLayoutScanWithTypeOverride schema - {}", res.getPartitions().getSchema());
    logger.info("doGetTableLayoutScanWithTypeOverride partitions - {}", res.getPartitions());

    assertThat(res.getPartitions().getSchema().getCustomMetadata().get(PARTITION_TYPE_METADATA), equalTo(SCAN_PARTITION_TYPE));
    // no hash key constraints, so look for segment count column
    assertThat(res.getPartitions().getSchema().findField(SEGMENT_COUNT_METADATA) != null, is(true));
    assertThat(res.getPartitions().getRowCount(), equalTo(1));

    assertThat(res.getPartitions().getSchema().getCustomMetadata().get(NON_KEY_FILTER_METADATA), equalTo("(#col3 = :v0 OR attribute_not_exists(#col3) OR #col3 = :v1)"));

    ImmutableMap<String, String> expressionNames = ImmutableMap.of("#col3", "col3", "#col2", "col2");
    assertThat(res.getPartitions().getSchema().getCustomMetadata().get(EXPRESSION_NAMES_METADATA), equalTo(Jackson.toJsonString(expressionNames)));

    ImmutableMap<String, AttributeValue> expressionValues = ImmutableMap.of(":v0", ItemUtils.toAttributeValue(true), ":v1", ItemUtils.toAttributeValue(null));
    assertThat(res.getPartitions().getSchema().getCustomMetadata().get(EXPRESSION_VALUES_METADATA), equalTo(Jackson.toJsonString(expressionValues)));
}
Example #7
Source File: TestingMetastoreObjects.java    From presto with Apache License 2.0 4 votes vote down vote up 
public static io.prestosql.plugin.hive.metastore.Column getPrestoTestColumn()
{
    return new io.prestosql.plugin.hive.metastore.Column("test-col" + generateRandom(), HiveType.HIVE_STRING, Optional.of("column comment"));
}