org.apache.hadoop.hive.metastore.api.Partition Java Examples
The following examples show how to use
org.apache.hadoop.hive.metastore.api.Partition.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HdfsSnapshotLocationManager.java From circus-train with Apache License 2.0 | 6 votes |
static List<Path> calculateSubPaths( List<Partition> sourcePartitions, String sourceDataLocation, String copyBaseLocation) { List<Path> paths = new ArrayList<>(sourcePartitions.size()); for (Partition partition : sourcePartitions) { String partitionLocation = partition.getSd().getLocation(); String partitionBranch = partitionLocation.replace(sourceDataLocation, ""); while (partitionBranch.startsWith("/")) { partitionBranch = partitionBranch.substring(1); } Path copyPartitionPath = new Path(copyBaseLocation, partitionBranch); paths.add(copyPartitionPath); LOG.debug("Added sub-path {}.", copyPartitionPath.toString()); } return paths; }
Example #2
Source File: PartitionedTableMetadataUpdateReplication.java From circus-train with Apache License 2.0 | 6 votes |
private PartitionsAndStatistics filterOnReplicatedPartitions( CloseableMetaStoreClient replicaClient, PartitionsAndStatistics sourcePartitionsAndStatistics, List<FieldSchema> partitionKeys) throws TException { Map<Partition, ColumnStatistics> statisticsByPartition = new LinkedHashMap<>(); for (Partition partition : sourcePartitionsAndStatistics.getPartitions()) { try { replicaClient.getPartition(replicaDatabaseName, replicaTableName, partition.getValues()); statisticsByPartition.put(partition, sourcePartitionsAndStatistics.getStatisticsForPartition(partition)); } catch (NoSuchObjectException e) { LOG.debug("Partition {} doesn't exist, skipping it...", Warehouse.getQualifiedName(partition)); } } return new PartitionsAndStatistics(partitionKeys, statisticsByPartition); }
Example #3
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 6 votes |
/** * {@inheritDoc} */ @Override public int add_partitions_pspec(final List<PartitionSpec> newParts) throws TException { if (newParts == null || newParts.isEmpty()) { return 0; } final String dbName = newParts.get(0).getDbName(); final String tableName = newParts.get(0).getTableName(); return requestWrapper("add_partition", new Object[]{dbName, tableName}, () -> { final PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(newParts); final PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator(); final List<Partition> partitions = addPartitionsCore(dbName, tableName, Lists.newArrayList(partitionIterator), false); return partitions.size(); }); }
Example #4
Source File: DiffGeneratedPartitionPredicateTest.java From circus-train with Apache License 2.0 | 6 votes |
private void setupHiveTables() throws TException, IOException { List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("p1"), newFieldSchema("p2")); File tableLocation = new File("db1", "table1"); StorageDescriptor sd = newStorageDescriptor(tableLocation, "col0"); table1 = newTable("table1", "db1", partitionKeys, sd); Partition partition1 = newPartition(table1, "value1", "value2"); Partition partition2 = newPartition(table1, "value11", "value22"); table1Partitions = Arrays.asList(partition1, partition2); // table1PartitionNames = Arrays .asList(Warehouse.makePartName(partitionKeys, partition1.getValues()), Warehouse.makePartName(partitionKeys, partition2.getValues())); File tableLocation2 = new File("db2", "table2"); StorageDescriptor sd2 = newStorageDescriptor(tableLocation2, "col0"); table2 = newTable("table2", "db2", partitionKeys, sd2); }
Example #5
Source File: InMemoryThriftMetastore.java From presto with Apache License 2.0 | 6 votes |
private static boolean partitionMatches(Partition partition, String databaseName, String tableName, List<String> parts) { if (!partition.getDbName().equals(databaseName) || !partition.getTableName().equals(tableName)) { return false; } List<String> values = partition.getValues(); if (values.size() != parts.size()) { return false; } for (int i = 0; i < values.size(); i++) { String part = parts.get(i); if (!part.isEmpty() && !values.get(i).equals(part)) { return false; } } return true; }
Example #6
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 6 votes |
/** * {@inheritDoc} */ @Override public void alter_partition_with_environment_context( final String dbName, final String tblName, final Partition newPart, @Nullable final EnvironmentContext ec ) throws TException { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); requestWrapper("alter_partition_with_environment_context", new Object[]{databaseName, tableName, ec}, () -> { addPartitionsCore(dbName, tableName, ImmutableList.of(newPart), false); return null; }); }
Example #7
Source File: HdfsSnapshotLocationManager.java From circus-train with Apache License 2.0 | 6 votes |
HdfsSnapshotLocationManager( HiveConf sourceHiveConf, String eventId, Table sourceTable, List<Partition> sourcePartitions, boolean snapshotsDisabled, String tableBasePath, FileSystemFactory fileSystemFactory, SourceCatalogListener sourceCatalogListener) throws IOException { this.sourceHiveConf = sourceHiveConf; this.eventId = eventId; this.sourceTable = sourceTable; this.snapshotsDisabled = snapshotsDisabled; this.sourceCatalogListener = sourceCatalogListener; this.fileSystemFactory = fileSystemFactory; String sourceDataLocation; if (StringUtils.isNotBlank(tableBasePath)) { sourceDataLocation = tableBasePath; } else { sourceDataLocation = sourceTable.getSd().getLocation(); } sourceDataPath = new Path(sourceDataLocation); copyBasePath = createSnapshot(); String copyBaseLocation = copyBasePath.toString(); subPaths = calculateSubPaths(sourcePartitions, sourceDataLocation, copyBaseLocation); }
Example #8
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 6 votes |
private List<Partition> getPartitionsByFilter(final String dbName, final String tblName, @Nullable final String filter, final short maxParts) { final String databaseName = normalizeIdentifier(dbName); final String tableName = normalizeIdentifier(tblName); final TableDto tableDto = v1.getTable(catalogName, databaseName, tableName, true, false, false); final Integer maxValues = maxParts > 0 ? Short.toUnsignedInt(maxParts) : null; final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(filter, null, true, false); final List<PartitionDto> metacatPartitions = partV1.getPartitionsForRequest(catalogName, dbName, tblName, null, null, null, maxValues, false, dto); final List<Partition> result = Lists.newArrayListWithCapacity(metacatPartitions.size()); for (PartitionDto partition : metacatPartitions) { result.add(hiveConverters.metacatToHivePartition(partition, tableDto)); } return result; }
Example #9
Source File: PartitionedTableReplicationTest.java From circus-train with Apache License 2.0 | 6 votes |
@Test public void noMatchingPartitions() throws Exception { when(replica.getLocationManager(TableType.PARTITIONED, targetTableLocation, EVENT_ID, sourceLocationManager)) .thenReturn(replicaLocationManager); PartitionsAndStatistics emptyPartitionsAndStats = new PartitionsAndStatistics(sourceTable.getPartitionKeys(), Collections.<Partition>emptyList(), Collections.<String, List<ColumnStatisticsObj>>emptyMap()); when(source.getPartitions(sourceTable, PARTITION_PREDICATE, MAX_PARTITIONS)).thenReturn(emptyPartitionsAndStats); when(source.getLocationManager(sourceTable, Collections.<Partition>emptyList(), EVENT_ID, copierOptions)) .thenReturn(sourceLocationManager); PartitionedTableReplication replication = new PartitionedTableReplication(DATABASE, TABLE, partitionPredicate, source, replica, copierFactoryManager, eventIdFactory, targetTableLocation, DATABASE, TABLE, copierOptions, listener, dataManipulatorFactoryManager); replication.replicate(); verifyZeroInteractions(copier); InOrder replicationOrder = inOrder(sourceLocationManager, replica, replicaLocationManager, listener); replicationOrder.verify(replica).validateReplicaTable(DATABASE, TABLE); replicationOrder .verify(replica) .updateMetadata(EVENT_ID, sourceTableAndStatistics, DATABASE, TABLE, replicaLocationManager); }
Example #10
Source File: HiveDifferencesIntegrationTest.java From circus-train with Apache License 2.0 | 6 votes |
@Test public void tablesAreDifferent() throws Exception { Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE); sourceTable.getParameters().put("com.company.team", "value"); catalog.client().alter_table(DATABASE, SOURCE_TABLE, sourceTable); // Reload table object sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE); Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE); HiveDifferences .builder(diffListener) .comparatorRegistry(comparatorRegistry) .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE)) .replica(Optional.of(replicaTable), Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE))) .checksumFunction(checksumFunction) .build() .run(); verify(diffListener, times(1)).onChangedTable(anyList()); verify(diffListener, never()).onNewPartition(anyString(), any(Partition.class)); verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList()); verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class)); }
Example #11
Source File: HivePartitionManagerTest.java From data-highway with Apache License 2.0 | 6 votes |
@Test public void addPartition() throws Exception { doReturn(addedPartition).when(metaStoreClient).add_partition(any()); doReturn(URI.create("resolved/location")).when(locationResolver).resolveLocation(LOCATION, false); doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant(); Partition result = underTest.addPartition(TABLE, PARTITION_VALUES, LOCATION).get(); ArgumentCaptor<Partition> captor = ArgumentCaptor.forClass(Partition.class); verify(metaStoreClient).add_partition(captor.capture()); Partition partition = captor.getValue(); assertThat(partition.getParameters().get("data-highway.version"), is(DataHighwayVersion.VERSION)); assertThat(partition.getParameters().get("data-highway.last-revision"), is("2018-05-16T09:17:05Z")); assertThat(partition.getSd().getLocation(), is("resolved/location")); assertThat(result, is(addedPartition)); }
Example #12
Source File: ReplicaTableFactoryTest.java From circus-train with Apache License 2.0 | 6 votes |
@Test public void newPartition() { Path replicaPartitionPath = new Path(REPLICA_DATA_DESTINATION, REPLICA_PARTITION_SUBPATH); Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME, replicaPartitionPath, FULL); assertThat(replica.getDbName(), is(sourceTable.getDbName())); assertThat(replica.getTableName(), is(sourceTable.getTableName())); assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT)); assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT)); assertThat(replica.getSd().getLocation(), is(replicaPartitionPath.toUri().toString())); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"), is(SOURCE_META_STORE_URIS)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(PARTITION_LOCATION)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue()))); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name())); assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true")); assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true")); assertThat(replica.getParameters().get("STATS_GENERATED"), is("true")); assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1")); }
Example #13
Source File: HiveMetaStoreUtils.java From incubator-gobblin with Apache License 2.0 | 6 votes |
/** * Convert a {@link HivePartition} into a {@link Partition}. */ public static Partition getPartition(HivePartition hivePartition) { State props = hivePartition.getProps(); Partition partition = new Partition(); partition.setDbName(hivePartition.getDbName()); partition.setTableName(hivePartition.getTableName()); partition.setValues(hivePartition.getValues()); partition.setParameters(getParameters(props)); if (hivePartition.getCreateTime().isPresent()) { partition.setCreateTime(Ints.checkedCast(hivePartition.getCreateTime().get())); } else if (props.contains(HiveConstants.CREATE_TIME)) { partition.setCreateTime(props.getPropAsInt(HiveConstants.CREATE_TIME)); } if (props.contains(HiveConstants.LAST_ACCESS_TIME)) { partition.setLastAccessTime(props.getPropAsInt(HiveConstants.LAST_ACCESS_TIME)); } partition.setSd(getStorageDescriptor(hivePartition)); return partition; }
Example #14
Source File: LanderTaskRunner.java From data-highway with Apache License 2.0 | 6 votes |
void updateMetadata(LanderConfiguration config) { changeState(State.UPDATING); String acquisitionInstant = config.getAcquisitionInstant(); log.info("Updating table to add partition {}, {}.", config.getRoadName(), acquisitionInstant); String partitionSpec = ACQUISITION_INSTANT + "=" + acquisitionInstant; List<String> partitionValues = singletonList(acquisitionInstant); try { Optional<Partition> partition = hivePartitionManager.addPartition(roadName, partitionValues, config.getS3KeyPrefix()); Map<Integer, Long> offsets = new HashMap<>(); config.getOffsets().forEach((pid, range) -> offsets.put(pid, range.getEnd())); offsetManager.commitOffsets(topicName, offsets); if (partition.isPresent()) { long recordCount = config.getOffsets().values().stream().mapToLong(r -> r.getEnd() - r.getStart()).sum(); landingHandler.handlePartitionCreated(roadName, partition.get(), partitionSpec, recordCount); } else { // Partition already exists partitionMutationCounter.increment(); log.warn("Data landed into existing partition; road={} partitionSpec={}", roadName, partitionSpec); } } catch (MetaStoreException e) { metaStoreErrorMeter.increment(); throw e; } }
Example #15
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 6 votes |
public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) { if (partition == null) { return Collections.emptyList(); } final List<String> partitionValues = partition.getValues(); final List<PartitionValue> output = new ArrayList<>(); final List<FieldSchema> partitionKeys = table.getPartitionKeys(); for (int i = 0; i < partitionKeys.size(); i++) { final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth); if (value != null) { output.add(value); } } return output; }
Example #16
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 6 votes |
/** * {@inheritDoc} */ @Override public Partition append_partition_with_environment_context( final String dbName, final String tblName, final List<String> partVals, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("append_partition_by_name_with_environment_context", new Object[]{dbName, tblName, partVals}, () -> { final TableDto tableDto = getTableDto(dbName, tblName); final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals); appendPartitionsCore(dbName, tblName, partName); return hiveConverters.metacatToHivePartition(getPartitionDtoByName(tableDto, partName), tableDto); }); }
Example #17
Source File: PartitionsAndStatisticsTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void getStatisticsForPartitionReturnsNullIfEmptyStats() throws Exception { List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a")); Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1")); List<Partition> partitions = Lists.newArrayList(newPartition(table, "b")); statisticsPerPartitionName.put("a=b", Collections.<ColumnStatisticsObj> emptyList()); PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions, statisticsPerPartitionName); assertNull(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0))); }
Example #18
Source File: AuthorizingObjectStore.java From incubator-sentry with Apache License 2.0 | 5 votes |
@Override public List<Partition> listPartitionsPsWithAuth(String dbName, String tblName, List<String> part_vals, short max_parts, String userName, List<String> groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) { throw new MetaException(getNoAccessMessageForTable(dbName, tblName)); } return super.listPartitionsPsWithAuth(dbName, tblName, part_vals, max_parts, userName, groupNames); }
Example #19
Source File: HiveMetaStoreBasedRegister.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private void addOrAlterPartitionWithPushMode(IMetaStoreClient client, Table table, HivePartition partition) throws TException, IOException { Partition nativePartition = HiveMetaStoreUtils.getPartition(partition); Preconditions.checkArgument(table.getPartitionKeysSize() == nativePartition.getValues().size(), String.format("Partition key size is %s but partition value size is %s", table.getPartitionKeys().size(), nativePartition.getValues().size())); try (AutoCloseableHiveLock lock = this.locks.getPartitionLock(table.getDbName(), table.getTableName(), nativePartition.getValues())) { try { try (Timer.Context context = this.metricContext.timer(ADD_PARTITION_TIMER).time()) { client.add_partition(getPartitionWithCreateTimeNow(nativePartition)); } log.info(String.format("Added partition %s to table %s with location %s", stringifyPartition(nativePartition), table.getTableName(), nativePartition.getSd().getLocation())); } catch (TException e) { try { if (this.skipDiffComputation) { onPartitionExistWithoutComputingDiff(table, nativePartition, e); } else { onPartitionExist(client, table, partition, nativePartition, null); } } catch (Throwable e2) { log.error(String.format( "Unable to add or alter partition %s in table %s with location %s: " + e2.getMessage(), stringifyPartitionVerbose(nativePartition), table.getTableName(), nativePartition.getSd().getLocation()), e2); throw e2; } } } }
Example #20
Source File: HiveDataFragmenter.java From pxf with Apache License 2.0 | 5 votes |
private void fetchMetaDataForPartitionedTable(StorageDescriptor stdsc, Properties props, Partition partition, List<FieldSchema> partitionKeys, String tableName, boolean hasComplexTypes, List<Integer> hiveIndexes, String allColumnNames, String allColumnTypes) throws Exception { fetchMetaData(new HiveTablePartition(stdsc, props, partition, partitionKeys, tableName), hasComplexTypes, hiveIndexes, allColumnNames, allColumnTypes); }
Example #21
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void get_partition() throws TException { List<String> partVals = Lists.newArrayList(); Partition partition = new Partition(); Partition outbound = new Partition(); when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); when(primaryClient.get_partition("inbound", "table1", partVals)).thenReturn(partition); when(primaryMapping.transformOutboundPartition(partition)).thenReturn(outbound); Partition result = handler.get_partition(DB_P, "table1", partVals); assertThat(result, is(outbound)); verify(primaryMapping, never()).checkWritePermissions(DB_P); }
Example #22
Source File: MetacatHMSHandler.java From metacat with Apache License 2.0 | 5 votes |
private boolean startAddPartition( final RawStore ms, final Partition part, final boolean ifNotExists) throws MetaException, TException { MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); final boolean doesExist = ms.doesPartitionExist( part.getDbName(), part.getTableName(), part.getValues()); if (doesExist && !ifNotExists) { throw new AlreadyExistsException("Partition already exists: " + part); } return !doesExist; }
Example #23
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void append_partition() throws TException { Partition inbound = new Partition(); Partition outbound = new Partition(); List<String> partVals = Lists.newArrayList(); when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound"); when(primaryClient.append_partition("inbound", "table1", partVals)).thenReturn(inbound); when(primaryMapping.transformOutboundPartition(inbound)).thenReturn(outbound); Partition result = handler.append_partition(DB_P, "table1", partVals); assertThat(result, is(outbound)); verify(primaryMapping).checkWritePermissions(DB_P); }
Example #24
Source File: HiveConvertExtractor.java From incubator-gobblin with Apache License 2.0 | 5 votes |
public HiveConvertExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException { super(state); if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY)); return; } if (!(this.hiveDataset instanceof ConvertibleHiveDataset)) { throw new IllegalStateException("HiveConvertExtractor is only compatible with ConvertibleHiveDataset"); } ConvertibleHiveDataset convertibleHiveDataset = (ConvertibleHiveDataset) this.hiveDataset; try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) { Table table = client.get().getTable(this.dbName, this.tableName); SchemaAwareHiveTable schemaAwareHiveTable = new SchemaAwareHiveTable(table, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getTableSchemaUrl(), fs)); SchemaAwareHivePartition schemaAwareHivePartition = null; if (this.hiveWorkUnit.getPartitionName().isPresent() && this.hiveWorkUnit.getPartitionSchemaUrl().isPresent()) { Partition partition = client.get().getPartition(this.dbName, this.tableName, this.hiveWorkUnit.getPartitionName().get()); schemaAwareHivePartition = new SchemaAwareHivePartition(table, partition, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getPartitionSchemaUrl().get(), fs)); } QueryBasedHiveConversionEntity entity = new QueryBasedHiveConversionEntity(convertibleHiveDataset, schemaAwareHiveTable, Optional.fromNullable(schemaAwareHivePartition)); this.conversionEntities.add(entity); } }
Example #25
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void add_partitions() throws TException { Partition newPartition1 = new Partition(); newPartition1.setDbName(DB_P); Partition newPartition2 = new Partition(); newPartition2.setDbName(DB_P); List<Partition> inbound = Lists.newArrayList(new Partition()); List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2); when(primaryMapping.transformInboundPartitions(partitions)).thenReturn(inbound); when(primaryClient.add_partitions(inbound)).thenReturn(2); int result = handler.add_partitions(partitions); assertThat(result, is(2)); verify(primaryMapping, times(2)).checkWritePermissions(DB_P); }
Example #26
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void rename_partition() throws TException { Partition newPartition = new Partition(); newPartition.setDbName(DB_P); handler.rename_partition(DB_P, "table", Collections.emptyList(), newPartition); verify(primaryMapping, times(2)).checkWritePermissions(DB_P); verify(primaryClient).rename_partition(DB_P, "table", Collections.emptyList(), newPartition); }
Example #27
Source File: InMemoryThriftMetastore.java From presto with Apache License 2.0 | 5 votes |
@Override public synchronized void alterPartition(HiveIdentity identity, String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); }
Example #28
Source File: ComparisonToolIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition( String table, StorageDescriptor tableStorageDescriptor, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(table); partition.setValues(values); partition.setSd(new StorageDescriptor(tableStorageDescriptor)); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example #29
Source File: TestMetastoreEndToEnd.java From incubator-sentry with Apache License 2.0 | 5 votes |
private void verifyPartitionExists(String dbName, String tabName, String partVal) throws Exception { HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1); Partition newPartition = client.getPartition(dbName, tabName, Lists.newArrayList(partVal)); Assert.assertNotNull(newPartition); client.close(); }
Example #30
Source File: HiveDifferencesIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void sourcePartitionHasChanged() throws Exception { Partition sourcePartition1 = catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=1"); sourcePartition1.getSd().getCols().add(BAZ_COL); catalog.client().alter_partition(DATABASE, SOURCE_TABLE, sourcePartition1); Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE); Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE); HiveDifferences .builder(diffListener) .comparatorRegistry(comparatorRegistry) .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE)) .replica(Optional.of(replicaTable), Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE))) .checksumFunction(checksumFunction) .build() .run(); verify(diffListener, never()).onChangedTable(anyList()); verify(diffListener, never()).onNewPartition(anyString(), any(Partition.class)); verify(diffListener, times(1)) .onChangedPartition("part=1", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=1"), Arrays .<Diff<Object, Object>>asList(new BaseDiff<Object, Object>( "Collection partition.sd.cols of class java.util.ArrayList has different size: left.size()=3 and right.size()=2", Arrays.asList(FOO_COL, BAR_COL, BAZ_COL), Arrays.asList(FOO_COL, BAR_COL)))); verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class)); }