org.apache.hadoop.hive.metastore.api.Partition Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.Partition. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HdfsSnapshotLocationManager.java    From circus-train with Apache License 2.0 6 votes vote down vote up
static List<Path> calculateSubPaths(
    List<Partition> sourcePartitions,
    String sourceDataLocation,
    String copyBaseLocation) {
  List<Path> paths = new ArrayList<>(sourcePartitions.size());
  for (Partition partition : sourcePartitions) {
    String partitionLocation = partition.getSd().getLocation();
    String partitionBranch = partitionLocation.replace(sourceDataLocation, "");
    while (partitionBranch.startsWith("/")) {
      partitionBranch = partitionBranch.substring(1);
    }
    Path copyPartitionPath = new Path(copyBaseLocation, partitionBranch);
    paths.add(copyPartitionPath);
    LOG.debug("Added sub-path {}.", copyPartitionPath.toString());
  }
  return paths;
}
 
Example #2
Source File: PartitionedTableMetadataUpdateReplication.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private PartitionsAndStatistics filterOnReplicatedPartitions(
    CloseableMetaStoreClient replicaClient,
    PartitionsAndStatistics sourcePartitionsAndStatistics,
    List<FieldSchema> partitionKeys)
  throws TException {
  Map<Partition, ColumnStatistics> statisticsByPartition = new LinkedHashMap<>();
  for (Partition partition : sourcePartitionsAndStatistics.getPartitions()) {
    try {
      replicaClient.getPartition(replicaDatabaseName, replicaTableName, partition.getValues());
      statisticsByPartition.put(partition, sourcePartitionsAndStatistics.getStatisticsForPartition(partition));
    } catch (NoSuchObjectException e) {
      LOG.debug("Partition {} doesn't exist, skipping it...", Warehouse.getQualifiedName(partition));
    }
  }
  return new PartitionsAndStatistics(partitionKeys, statisticsByPartition);
}
 
Example #3
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public int add_partitions_pspec(final List<PartitionSpec> newParts) throws TException {
    if (newParts == null || newParts.isEmpty()) {
        return 0;
    }
    final String dbName = newParts.get(0).getDbName();
    final String tableName = newParts.get(0).getTableName();
    return requestWrapper("add_partition", new Object[]{dbName, tableName}, () -> {
        final PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(newParts);
        final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator();
        final List<Partition> partitions = addPartitionsCore(dbName, tableName,
            Lists.newArrayList(partitionIterator), false);
        return partitions.size();
    });
}
 
Example #4
Source File: DiffGeneratedPartitionPredicateTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private void setupHiveTables() throws TException, IOException {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("p1"), newFieldSchema("p2"));

  File tableLocation = new File("db1", "table1");
  StorageDescriptor sd = newStorageDescriptor(tableLocation, "col0");
  table1 = newTable("table1", "db1", partitionKeys, sd);
  Partition partition1 = newPartition(table1, "value1", "value2");
  Partition partition2 = newPartition(table1, "value11", "value22");
  table1Partitions = Arrays.asList(partition1, partition2); //
  table1PartitionNames = Arrays
      .asList(Warehouse.makePartName(partitionKeys, partition1.getValues()),
          Warehouse.makePartName(partitionKeys, partition2.getValues()));

  File tableLocation2 = new File("db2", "table2");
  StorageDescriptor sd2 = newStorageDescriptor(tableLocation2, "col0");
  table2 = newTable("table2", "db2", partitionKeys, sd2);
}
 
Example #5
Source File: InMemoryThriftMetastore.java    From presto with Apache License 2.0 6 votes vote down vote up
private static boolean partitionMatches(Partition partition, String databaseName, String tableName, List<String> parts)
{
    if (!partition.getDbName().equals(databaseName) ||
            !partition.getTableName().equals(tableName)) {
        return false;
    }
    List<String> values = partition.getValues();
    if (values.size() != parts.size()) {
        return false;
    }
    for (int i = 0; i < values.size(); i++) {
        String part = parts.get(i);
        if (!part.isEmpty() && !values.get(i).equals(part)) {
            return false;
        }
    }
    return true;
}
 
Example #6
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void alter_partition_with_environment_context(
    final String dbName,
    final String tblName,
    final Partition newPart,
    @Nullable final EnvironmentContext ec
) throws TException {
    final String databaseName = normalizeIdentifier(dbName);
    final String tableName = normalizeIdentifier(tblName);
    requestWrapper("alter_partition_with_environment_context", new Object[]{databaseName, tableName, ec},
        () -> {
            addPartitionsCore(dbName, tableName, ImmutableList.of(newPart), false);
            return null;
        });
}
 
Example #7
Source File: HdfsSnapshotLocationManager.java    From circus-train with Apache License 2.0 6 votes vote down vote up
HdfsSnapshotLocationManager(
    HiveConf sourceHiveConf,
    String eventId,
    Table sourceTable,
    List<Partition> sourcePartitions,
    boolean snapshotsDisabled,
    String tableBasePath,
    FileSystemFactory fileSystemFactory,
    SourceCatalogListener sourceCatalogListener) throws IOException {
  this.sourceHiveConf = sourceHiveConf;
  this.eventId = eventId;
  this.sourceTable = sourceTable;
  this.snapshotsDisabled = snapshotsDisabled;
  this.sourceCatalogListener = sourceCatalogListener;
  this.fileSystemFactory = fileSystemFactory;
  String sourceDataLocation;
  if (StringUtils.isNotBlank(tableBasePath)) {
    sourceDataLocation = tableBasePath;
  } else {
    sourceDataLocation = sourceTable.getSd().getLocation();
  }
  sourceDataPath = new Path(sourceDataLocation);
  copyBasePath = createSnapshot();
  String copyBaseLocation = copyBasePath.toString();
  subPaths = calculateSubPaths(sourcePartitions, sourceDataLocation, copyBaseLocation);
}
 
Example #8
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
private List<Partition> getPartitionsByFilter(final String dbName, final String tblName,
                                              @Nullable final String filter, final short maxParts) {
    final String databaseName = normalizeIdentifier(dbName);
    final String tableName = normalizeIdentifier(tblName);
    final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false);

    final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null;
    final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false);
    final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName,
        null, null, null, maxValues, false, dto);
    final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size());
    for (PartitionDto partition : metacatPartitions) {
        result.add(hiveConverters.metacatToHivePartition(partition, tableDto));
    }
    return result;
}
 
Example #9
Source File: PartitionedTableReplicationTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void noMatchingPartitions() throws Exception {
  when(replica.getLocationManager(TableType.PARTITIONED, targetTableLocation, EVENT_ID, sourceLocationManager))
      .thenReturn(replicaLocationManager);
  PartitionsAndStatistics emptyPartitionsAndStats = new PartitionsAndStatistics(sourceTable.getPartitionKeys(),
      Collections.<Partition>emptyList(), Collections.<String, List<ColumnStatisticsObj>>emptyMap());
  when(source.getPartitions(sourceTable, PARTITION_PREDICATE, MAX_PARTITIONS)).thenReturn(emptyPartitionsAndStats);
  when(source.getLocationManager(sourceTable, Collections.<Partition>emptyList(), EVENT_ID, copierOptions))
      .thenReturn(sourceLocationManager);

  PartitionedTableReplication replication = new PartitionedTableReplication(DATABASE, TABLE, partitionPredicate,
      source, replica, copierFactoryManager, eventIdFactory, targetTableLocation, DATABASE, TABLE, copierOptions,
      listener, dataManipulatorFactoryManager);
  replication.replicate();

  verifyZeroInteractions(copier);
  InOrder replicationOrder = inOrder(sourceLocationManager, replica, replicaLocationManager, listener);
  replicationOrder.verify(replica).validateReplicaTable(DATABASE, TABLE);
  replicationOrder
      .verify(replica)
      .updateMetadata(EVENT_ID, sourceTableAndStatistics, DATABASE, TABLE, replicaLocationManager);
}
 
Example #10
Source File: HiveDifferencesIntegrationTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void tablesAreDifferent() throws Exception {
  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  sourceTable.getParameters().put("com.company.team", "value");
  catalog.client().alter_table(DATABASE, SOURCE_TABLE, sourceTable);

  // Reload table object
  sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);

  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();
  verify(diffListener, times(1)).onChangedTable(anyList());
  verify(diffListener, never()).onNewPartition(anyString(), any(Partition.class));
  verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList());
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}
 
Example #11
Source File: HivePartitionManagerTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Test
public void addPartition() throws Exception {
  doReturn(addedPartition).when(metaStoreClient).add_partition(any());
  doReturn(URI.create("resolved/location")).when(locationResolver).resolveLocation(LOCATION, false);
  doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant();

  Partition result = underTest.addPartition(TABLE, PARTITION_VALUES, LOCATION).get();

  ArgumentCaptor<Partition> captor = ArgumentCaptor.forClass(Partition.class);
  verify(metaStoreClient).add_partition(captor.capture());

  Partition partition = captor.getValue();
  assertThat(partition.getParameters().get("data-highway.version"), is(DataHighwayVersion.VERSION));
  assertThat(partition.getParameters().get("data-highway.last-revision"), is("2018-05-16T09:17:05Z"));
  assertThat(partition.getSd().getLocation(), is("resolved/location"));

  assertThat(result, is(addedPartition));
}
 
Example #12
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void newPartition() {
  Path replicaPartitionPath = new Path(REPLICA_DATA_DESTINATION, REPLICA_PARTITION_SUBPATH);
  Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME,
      replicaPartitionPath, FULL);

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(replicaPartitionPath.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(PARTITION_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
}
 
Example #13
Source File: HiveMetaStoreUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * Convert a {@link HivePartition} into a {@link Partition}.
 */
public static Partition getPartition(HivePartition hivePartition) {
  State props = hivePartition.getProps();
  Partition partition = new Partition();
  partition.setDbName(hivePartition.getDbName());
  partition.setTableName(hivePartition.getTableName());
  partition.setValues(hivePartition.getValues());
  partition.setParameters(getParameters(props));
  if (hivePartition.getCreateTime().isPresent()) {
    partition.setCreateTime(Ints.checkedCast(hivePartition.getCreateTime().get()));
  } else if (props.contains(HiveConstants.CREATE_TIME)) {
    partition.setCreateTime(props.getPropAsInt(HiveConstants.CREATE_TIME));
  }
  if (props.contains(HiveConstants.LAST_ACCESS_TIME)) {
    partition.setLastAccessTime(props.getPropAsInt(HiveConstants.LAST_ACCESS_TIME));
  }
  partition.setSd(getStorageDescriptor(hivePartition));
  return partition;
}
 
Example #14
Source File: LanderTaskRunner.java    From data-highway with Apache License 2.0 6 votes vote down vote up
void updateMetadata(LanderConfiguration config) {
  changeState(State.UPDATING);
  String acquisitionInstant = config.getAcquisitionInstant();
  log.info("Updating table to add partition {}, {}.", config.getRoadName(), acquisitionInstant);

  String partitionSpec = ACQUISITION_INSTANT + "=" + acquisitionInstant;
  List<String> partitionValues = singletonList(acquisitionInstant);
  try {
    Optional<Partition> partition = hivePartitionManager.addPartition(roadName, partitionValues,
        config.getS3KeyPrefix());
    Map<Integer, Long> offsets = new HashMap<>();
    config.getOffsets().forEach((pid, range) -> offsets.put(pid, range.getEnd()));
    offsetManager.commitOffsets(topicName, offsets);
    if (partition.isPresent()) {
      long recordCount = config.getOffsets().values().stream().mapToLong(r -> r.getEnd() - r.getStart()).sum();
      landingHandler.handlePartitionCreated(roadName, partition.get(), partitionSpec, recordCount);
    } else {
      // Partition already exists
      partitionMutationCounter.increment();
      log.warn("Data landed into existing partition; road={} partitionSpec={}", roadName, partitionSpec);
    }
  } catch (MetaStoreException e) {
    metaStoreErrorMeter.increment();
    throw e;
  }
}
 
Example #15
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 
Example #16
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public Partition append_partition_with_environment_context(
    final String dbName,
    final String tblName,
    final List<String> partVals,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("append_partition_by_name_with_environment_context",
        new Object[]{dbName, tblName, partVals}, () -> {
            final TableDto tableDto = getTableDto(dbName, tblName);
            final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals);
            appendPartitionsCore(dbName, tblName, partName);
            return hiveConverters.metacatToHivePartition(getPartitionDtoByName(tableDto, partName), tableDto);
        });
}
 
Example #17
Source File: PartitionsAndStatisticsTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void getStatisticsForPartitionReturnsNullIfEmptyStats() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"));
  Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1"));
  List<Partition> partitions = Lists.newArrayList(newPartition(table, "b"));
  statisticsPerPartitionName.put("a=b", Collections.<ColumnStatisticsObj> emptyList());

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);

  assertNull(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0)));
}
 
Example #18
Source File: AuthorizingObjectStore.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public List<Partition> listPartitionsPsWithAuth(String dbName,
    String tblName, List<String> part_vals, short max_parts, String userName,
    List<String> groupNames) throws MetaException, InvalidObjectException,
    NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tblName));
  }
  return super.listPartitionsPsWithAuth(dbName, tblName, part_vals,
      max_parts, userName, groupNames);
}
 
Example #19
Source File: HiveMetaStoreBasedRegister.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private void addOrAlterPartitionWithPushMode(IMetaStoreClient client, Table table, HivePartition partition)
    throws TException, IOException {
  Partition nativePartition = HiveMetaStoreUtils.getPartition(partition);

  Preconditions.checkArgument(table.getPartitionKeysSize() == nativePartition.getValues().size(),
      String.format("Partition key size is %s but partition value size is %s", table.getPartitionKeys().size(),
          nativePartition.getValues().size()));

  try (AutoCloseableHiveLock lock =
      this.locks.getPartitionLock(table.getDbName(), table.getTableName(), nativePartition.getValues())) {

    try {
      try (Timer.Context context = this.metricContext.timer(ADD_PARTITION_TIMER).time()) {
        client.add_partition(getPartitionWithCreateTimeNow(nativePartition));
      }
      log.info(String.format("Added partition %s to table %s with location %s", stringifyPartition(nativePartition),
          table.getTableName(), nativePartition.getSd().getLocation()));
    } catch (TException e) {
      try {
        if (this.skipDiffComputation) {
          onPartitionExistWithoutComputingDiff(table, nativePartition, e);
        } else {
          onPartitionExist(client, table, partition, nativePartition, null);
        }
      } catch (Throwable e2) {
        log.error(String.format(
            "Unable to add or alter partition %s in table %s with location %s: " + e2.getMessage(),
            stringifyPartitionVerbose(nativePartition), table.getTableName(), nativePartition.getSd().getLocation()), e2);
        throw e2;
      }
    }
  }
}
 
Example #20
Source File: HiveDataFragmenter.java    From pxf with Apache License 2.0 5 votes vote down vote up
private void fetchMetaDataForPartitionedTable(StorageDescriptor stdsc,
                                              Properties props,
                                              Partition partition,
                                              List<FieldSchema> partitionKeys,
                                              String tableName,
                                              boolean hasComplexTypes,
                                              List<Integer> hiveIndexes,
                                              String allColumnNames,
                                              String allColumnTypes)
        throws Exception {
    fetchMetaData(new HiveTablePartition(stdsc, props, partition,
            partitionKeys, tableName), hasComplexTypes, hiveIndexes,
            allColumnNames, allColumnTypes);
}
 
Example #21
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_partition() throws TException {
  List<String> partVals = Lists.newArrayList();
  Partition partition = new Partition();
  Partition outbound = new Partition();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.get_partition("inbound", "table1", partVals)).thenReturn(partition);
  when(primaryMapping.transformOutboundPartition(partition)).thenReturn(outbound);
  Partition result = handler.get_partition(DB_P, "table1", partVals);
  assertThat(result, is(outbound));
  verify(primaryMapping, never()).checkWritePermissions(DB_P);
}
 
Example #22
Source File: MetacatHMSHandler.java    From metacat with Apache License 2.0 5 votes vote down vote up
private boolean startAddPartition(
        final RawStore ms, final Partition part, final boolean ifNotExists) throws MetaException, TException {
    MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
            partitionValidationPattern);
    final boolean doesExist = ms.doesPartitionExist(
            part.getDbName(), part.getTableName(), part.getValues());
    if (doesExist && !ifNotExists) {
        throw new AlreadyExistsException("Partition already exists: " + part);
    }
    return !doesExist;
}
 
Example #23
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void append_partition() throws TException {
  Partition inbound = new Partition();
  Partition outbound = new Partition();
  List<String> partVals = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.append_partition("inbound", "table1", partVals)).thenReturn(inbound);
  when(primaryMapping.transformOutboundPartition(inbound)).thenReturn(outbound);
  Partition result = handler.append_partition(DB_P, "table1", partVals);
  assertThat(result, is(outbound));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #24
Source File: HiveConvertExtractor.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
public HiveConvertExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException {
  super(state);

  if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
    log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY));
    return;
  }

  if (!(this.hiveDataset instanceof ConvertibleHiveDataset)) {
    throw new IllegalStateException("HiveConvertExtractor is only compatible with ConvertibleHiveDataset");
  }

  ConvertibleHiveDataset convertibleHiveDataset = (ConvertibleHiveDataset) this.hiveDataset;

  try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) {
    Table table = client.get().getTable(this.dbName, this.tableName);

    SchemaAwareHiveTable schemaAwareHiveTable = new SchemaAwareHiveTable(table, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getTableSchemaUrl(), fs));

    SchemaAwareHivePartition schemaAwareHivePartition = null;

    if (this.hiveWorkUnit.getPartitionName().isPresent() && this.hiveWorkUnit.getPartitionSchemaUrl().isPresent()) {

      Partition partition = client.get().getPartition(this.dbName, this.tableName, this.hiveWorkUnit.getPartitionName().get());
      schemaAwareHivePartition =
          new SchemaAwareHivePartition(table, partition, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getPartitionSchemaUrl().get(), fs));
    }

    QueryBasedHiveConversionEntity entity =
        new QueryBasedHiveConversionEntity(convertibleHiveDataset, schemaAwareHiveTable, Optional.fromNullable(schemaAwareHivePartition));
    this.conversionEntities.add(entity);
  }

}
 
Example #25
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void add_partitions() throws TException {
  Partition newPartition1 = new Partition();
  newPartition1.setDbName(DB_P);
  Partition newPartition2 = new Partition();
  newPartition2.setDbName(DB_P);
  List<Partition> inbound = Lists.newArrayList(new Partition());
  List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2);
  when(primaryMapping.transformInboundPartitions(partitions)).thenReturn(inbound);
  when(primaryClient.add_partitions(inbound)).thenReturn(2);
  int result = handler.add_partitions(partitions);
  assertThat(result, is(2));
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
}
 
Example #26
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void rename_partition() throws TException {
  Partition newPartition = new Partition();
  newPartition.setDbName(DB_P);
  handler.rename_partition(DB_P, "table", Collections.emptyList(), newPartition);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).rename_partition(DB_P, "table", Collections.emptyList(), newPartition);
}
 
Example #27
Source File: InMemoryThriftMetastore.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void alterPartition(HiveIdentity identity, String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics)
{
    Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition());
    if (partition.getParameters() == null) {
        partition.setParameters(ImmutableMap.of());
    }
    PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName());
    partitions.put(partitionKey, partition);
    partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics());
}
 
Example #28
Source File: ComparisonToolIntegrationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private Partition newPartition(
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 
Example #29
Source File: TestMetastoreEndToEnd.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
private void verifyPartitionExists(String dbName, String tabName,
    String partVal) throws Exception {
  HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
  Partition newPartition = client.getPartition(dbName, tabName,
      Lists.newArrayList(partVal));
  Assert.assertNotNull(newPartition);
  client.close();
}
 
Example #30
Source File: HiveDifferencesIntegrationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void sourcePartitionHasChanged() throws Exception {
  Partition sourcePartition1 = catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=1");
  sourcePartition1.getSd().getCols().add(BAZ_COL);
  catalog.client().alter_partition(DATABASE, SOURCE_TABLE, sourcePartition1);

  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);
  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();

  verify(diffListener, never()).onChangedTable(anyList());
  verify(diffListener, never()).onNewPartition(anyString(), any(Partition.class));
  verify(diffListener, times(1))
      .onChangedPartition("part=1", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=1"), Arrays
          .<Diff<Object, Object>>asList(new BaseDiff<Object, Object>(
              "Collection partition.sd.cols of class java.util.ArrayList has different size: left.size()=3 and right.size()=2",
              Arrays.asList(FOO_COL, BAR_COL, BAZ_COL), Arrays.asList(FOO_COL, BAR_COL))));
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}