org.apache.hadoop.hive.metastore.api.PartitionSpec Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.PartitionSpec. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public int add_partitions_pspec(final List<PartitionSpec> newParts) throws TException {
    if (newParts == null || newParts.isEmpty()) {
        return 0;
    }
    final String dbName = newParts.get(0).getDbName();
    final String tableName = newParts.get(0).getTableName();
    return requestWrapper("add_partition", new Object[]{dbName, tableName}, () -> {
        final PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(newParts);
        final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator();
        final List<Partition> partitions = addPartitionsCore(dbName, tableName,
            Lists.newArrayList(partitionIterator), false);
        return partitions.size();
    });
}
 
Example #2
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  for (PartitionSpec partitionSpec : partitionSpecs) {
    transformOutboundPartitionSpec(partitionSpec);
  }
  return partitionSpecs;
}
 
Example #3
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  for (PartitionSpec partitionSpec : partitionSpecs) {
    partitionSpec.setDbName(metaStoreMapping.transformInboundDatabaseName(partitionSpec.getDbName()));
  }
  return partitionSpecs;
}
 
Example #4
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName,
                                                    final String filter, final int maxParts)
    throws TException {
    //TODO: Handle the use case of grouping
    return requestWrapper("get_partitions_pspec", new Object[]{dbName, tblName, filter, maxParts}, () -> {
        final String databaseName = normalizeIdentifier(dbName);
        final String tableName = normalizeIdentifier(tblName);
        final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false);
        final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false);
        final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName,
            null, null, null, maxParts, false, dto);
        final List<Partition> partitions = Lists.newArrayListWithCapacity(metacatPartitions.size());
        for (PartitionDto partition : metacatPartitions) {
            partitions.add(hiveConverters.metacatToHivePartition(partition, tableDto));
        }

        final PartitionSpec pSpec = new PartitionSpec();
        pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
        pSpec.setDbName(dbName);
        pSpec.setTableName(tblName);
        if (tableDto != null && tableDto.getSerde() != null) {
            pSpec.setRootPath(tableDto.getSerde().getUri());
        }
        return Arrays.asList(pSpec);
    });
}
 
Example #5
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_part_specs_by_filter() throws TException {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList();
  List<PartitionSpec> outbound = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.get_part_specs_by_filter("inbound", "table", "*", (short) 10)).thenReturn(partitionSpecs);
  when(primaryMapping.transformOutboundPartitionSpecs(partitionSpecs)).thenReturn(outbound);
  List<PartitionSpec> result = handler.get_part_specs_by_filter(DB_P, "table", "*", (short) 10);
  assertThat(result, is(outbound));
  verify(primaryMapping, never()).checkWritePermissions(DB_P);
}
 
Example #6
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_partitions_pspec() throws TException {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList();
  List<PartitionSpec> outbound = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.get_partitions_pspec("inbound", "table", (short) 10)).thenReturn(partitionSpecs);
  when(primaryMapping.transformOutboundPartitionSpecs(partitionSpecs)).thenReturn(outbound);
  List<PartitionSpec> result = handler.get_partitions_pspec(DB_P, "table", (short) 10);
  assertThat(result, is(outbound));
  verify(primaryMapping, never()).checkWritePermissions(DB_P);
}
 
Example #7
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions_pspec(List<PartitionSpec> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (PartitionSpec partitionSpec : new_parts) {
      mapping.checkWritePermissions(partitionSpec.getDbName());
    }
    return mapping.getClient().add_partitions_pspec(mapping.transformInboundPartitionSpecs(new_parts));
  }
  return 0;
}
 
Example #8
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME, prepend=true)
public List<PartitionSpec> get_partitions_pspec(String db_name, String tbl_name, int max_parts)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  List<PartitionSpec> partitionSpecs = mapping
      .getClient()
      .get_partitions_pspec(mapping.transformInboundDatabaseName(db_name), tbl_name, max_parts);
  return mapping.transformOutboundPartitionSpecs(partitionSpecs);
}
 
Example #9
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME, prepend=true)
public List<PartitionSpec> get_part_specs_by_filter(String db_name, String tbl_name, String filter, int max_parts)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  List<PartitionSpec> partitionSpecs = mapping
      .getClient()
      .get_part_specs_by_filter(mapping.transformInboundDatabaseName(db_name), tbl_name, filter, max_parts);
  return mapping.transformOutboundPartitionSpecs(partitionSpecs);
}
 
Example #10
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void add_partitions_pspec() throws TException {
  PartitionSpec newPartitionPSpec1 = new PartitionSpec();
  newPartitionPSpec1.setDbName(DB_P);
  PartitionSpec newPartitionPspec2 = new PartitionSpec();
  newPartitionPspec2.setDbName(DB_P);
  List<PartitionSpec> inbound = Lists.newArrayList(new PartitionSpec());
  List<PartitionSpec> partitionsPspec = Lists.newArrayList(newPartitionPSpec1, newPartitionPspec2);
  when(primaryMapping.transformInboundPartitionSpecs(partitionsPspec)).thenReturn(inbound);
  when(primaryClient.add_partitions_pspec(inbound)).thenReturn(2);
  int result = handler.add_partitions_pspec(partitionsPspec);
  assertThat(result, is(2));
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
}
 
Example #11
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void transformInboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = Lists.newArrayList(partitionSpec);
  List<PartitionSpec> result = databaseMapping.transformInboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
  PartitionSpec resultSpec = result.get(0);
  assertThat(resultSpec, is(sameInstance(partitionSpec)));
  assertThat(resultSpec.getDbName(), is(IN_DB_NAME));
}
 
Example #12
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void transformOutboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  partitionSpecs.add(partitionSpec);
  List<PartitionSpec> result = databaseMapping.transformOutboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
  PartitionSpec resultSpec = result.get(0);
  assertThat(resultSpec, is(sameInstance(partitionSpec)));
  assertThat(resultSpec.getDbName(), is(OUT_DB_NAME));
}
 
Example #13
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  databaseMapping = new DatabaseMappingImpl(metastoreMapping, queryMapping);
  database = new Database();
  database.setName(DB_NAME);
  partition = new Partition();
  partition.setDbName(DB_NAME);
  partitions = Lists.newArrayList(partition);
  index = new Index();
  index.setDbName(DB_NAME);
  hiveObjectRef = new HiveObjectRef();
  hiveObjectRef.setDbName(DB_NAME);
  hiveObjectRef.setObjectType(HiveObjectType.DATABASE);
  hiveObjectRef.setObjectName(DB_NAME);
  hiveObjectPrivileges = new ArrayList<>();
  HiveObjectPrivilege hiveObjectPrivilege = new HiveObjectPrivilege();
  hiveObjectPrivilege.setHiveObject(hiveObjectRef);
  hiveObjectPrivileges.add(hiveObjectPrivilege);
  partitionSpec = new PartitionSpec();
  partitionSpec.setDbName(DB_NAME);
  when(metastoreMapping.transformInboundDatabaseName(anyString())).thenReturn(IN_DB_NAME);
  when(metastoreMapping.transformOutboundDatabaseName(anyString())).thenReturn(OUT_DB_NAME);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_EXPANDED_TEXT))
      .thenReturn(VIEW_EXPANDED_TEXT_TRANSFORMED);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_ORIGINAL_TEXT))
      .thenReturn(VIEW_ORIGINAL_TEXT_TRANSFORMED);
}
 
Example #14
Source File: SentryMetaStoreFilterHook.java    From incubator-sentry with Apache License 2.0 4 votes vote down vote up
@Override
public List<PartitionSpec> filterPartitionSpecs(
    List<PartitionSpec> partitionSpecList) {
  return partitionSpecList;
}
 
Example #15
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public PartitionSpec transformOutboundPartitionSpec(PartitionSpec partitionSpec) {
  partitionSpec.setDbName(metaStoreMapping.transformOutboundDatabaseName(partitionSpec.getDbName()));
  return partitionSpec;
}
 
Example #16
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 4 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<PartitionSpec> get_partitions_pspec(final String dbName, final String tblName, final int maxParts)
    throws TException {
    return get_part_specs_by_filter(dbName, tblName, null, maxParts);
}
 
Example #17
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Test
public void transformOutboundPartitionSpec() throws Exception {
  PartitionSpec result = databaseMapping.transformOutboundPartitionSpec(partitionSpec);
  assertThat(result, is(sameInstance(partitionSpec)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 
Example #18
Source File: IdentityMappingTest.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Test
public void transformInboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  List<PartitionSpec> result = databaseMapping.transformInboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
}
 
Example #19
Source File: IdentityMappingTest.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Test
public void transformOutboundPartitionSpecs() throws Exception {
  List<PartitionSpec> partitionSpecs = new ArrayList<>();
  List<PartitionSpec> result = databaseMapping.transformOutboundPartitionSpecs(partitionSpecs);
  assertThat(result, is(sameInstance(partitionSpecs)));
}
 
Example #20
Source File: IdentityMappingTest.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Test
public void transformOutboundPartitionSpec() throws Exception {
  PartitionSpec partitionSpec = new PartitionSpec();
  PartitionSpec result = databaseMapping.transformOutboundPartitionSpec(partitionSpec);
  assertThat(result, is(sameInstance(partitionSpec)));
}
 
Example #21
Source File: IdentityMapping.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  return partitionSpecs;
}
 
Example #22
Source File: IdentityMapping.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs) {
  return partitionSpecs;
}
 
Example #23
Source File: IdentityMapping.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public PartitionSpec transformOutboundPartitionSpec(PartitionSpec request) {
  return request;
}
 
Example #24
Source File: DatabaseMapping.java    From waggle-dance with Apache License 2.0 votes vote down vote up
List<PartitionSpec> transformInboundPartitionSpecs(List<PartitionSpec> partitionSpecs); 
Example #25
Source File: DatabaseMapping.java    From waggle-dance with Apache License 2.0 votes vote down vote up
List<PartitionSpec> transformOutboundPartitionSpecs(List<PartitionSpec> partitionSpecs); 
Example #26
Source File: DatabaseMapping.java    From waggle-dance with Apache License 2.0 votes vote down vote up
PartitionSpec transformOutboundPartitionSpec(PartitionSpec partitionSpec);