Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setValues()
The following examples show how to use
org.apache.hadoop.hive.metastore.api.Partition#setValues() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveTableOutputFormat.java From flink with Apache License 2.0 | 6 votes |
private void loadPartition(Path srcDir, Table table, Map<String, String> partSpec, HiveMetastoreClientWrapper client) throws TException, IOException { Path tblLocation = new Path(table.getSd().getLocation()); String dbName = tablePath.getDatabaseName(); String tableName = tablePath.getObjectName(); List<Partition> existingPart = client.listPartitions(dbName, tableName, new ArrayList<>(partSpec.values()), (short) 1); Path destDir = existingPart.isEmpty() ? new Path(tblLocation, Warehouse.makePartPath(partSpec)) : new Path(existingPart.get(0).getSd().getLocation()); moveFiles(srcDir, destDir); // register new partition if it doesn't exist if (existingPart.isEmpty()) { StorageDescriptor sd = new StorageDescriptor(hiveTablePartition.getStorageDescriptor()); sd.setLocation(destDir.toString()); Partition partition = HiveTableUtil.createHivePartition(dbName, tableName, new ArrayList<>(partSpec.values()), sd, new HashMap<>()); partition.setValues(new ArrayList<>(partSpec.values())); client.add_partition(partition); } }
Example 2
Source File: HiveAvroToOrcConverterTest.java From incubator-gobblin with Apache License 2.0 | 6 votes |
@Test public void dropReplacedPartitionsTest() throws Exception { Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName"); table.setTableType("VIRTUAL_VIEW"); table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", ""))); Partition part = new Partition(); part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01")); SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null); SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null); QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition)); List<ImmutableMap<String, String>> expected = ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01")); Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected); // Make sure that a partition itself is not dropped Partition replacedSelf = new Partition(); replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02")); replacedSelf.setValues(ImmutableList.of("2016", "02")); conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null))); Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected); }
Example 3
Source File: HiveMetaStoreUtils.java From incubator-gobblin with Apache License 2.0 | 6 votes |
/** * Convert a {@link HivePartition} into a {@link Partition}. */ public static Partition getPartition(HivePartition hivePartition) { State props = hivePartition.getProps(); Partition partition = new Partition(); partition.setDbName(hivePartition.getDbName()); partition.setTableName(hivePartition.getTableName()); partition.setValues(hivePartition.getValues()); partition.setParameters(getParameters(props)); if (hivePartition.getCreateTime().isPresent()) { partition.setCreateTime(Ints.checkedCast(hivePartition.getCreateTime().get())); } else if (props.contains(HiveConstants.CREATE_TIME)) { partition.setCreateTime(props.getPropAsInt(HiveConstants.CREATE_TIME)); } if (props.contains(HiveConstants.LAST_ACCESS_TIME)) { partition.setLastAccessTime(props.getPropAsInt(HiveConstants.LAST_ACCESS_TIME)); } partition.setSd(getStorageDescriptor(hivePartition)); return partition; }
Example 4
Source File: DestructiveReplicaTest.java From circus-train with Apache License 2.0 | 6 votes |
@Test public void dropDeletedPartitionsNothingToDrop() throws Exception { when(client.getTable(DATABASE, REPLICA_TABLE)).thenReturn(table); Partition replicaPartition = new Partition(); replicaPartition.setValues(Lists.newArrayList("value1")); List<Partition> replicaPartitions = Lists.newArrayList(replicaPartition); mockPartitionIterator(replicaPartitions); // source partition ("part1=value1") is the same as target partition nothing is deleted on source, nothing should be // deleted on target List<String> sourcePartitionNames = Lists.newArrayList("part1=value1"); replica.dropDeletedPartitions(sourcePartitionNames); verify(client, never()).dropPartition(DATABASE, REPLICA_TABLE, "part1=value1", false); verify(cleanupLocationManager, never()).addCleanupLocation(anyString(), any(Path.class)); verify(client).close(); verify(cleanupLocationManager).scheduleLocations(); }
Example 5
Source File: ReplicaTableFactoryTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void newReplicaPartitionStatistics() throws MetaException { sourceTable.setPartitionKeys( Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null))); Partition replicaPartition = new Partition(sourcePartition); replicaPartition.setDbName(MAPPED_DB_NAME); replicaPartition.setTableName(MAPPED_TABLE_NAME); replicaPartition.setValues(Arrays.asList("A", "B")); ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(); ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(); List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2); ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME); columnStatisticsDesc .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues())); ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs); ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition, sourcePartitionStatistics); assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME)); assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME)); assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B")); assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2)); assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1)); assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2)); }
Example 6
Source File: TestUtils.java From waggle-dance with Apache License 2.0 | 5 votes |
static Partition newPartition(Table hiveTable, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(hiveTable.getDbName()); partition.setTableName(hiveTable.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveTable.getSd())); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example 7
Source File: AbstractMetastoreTestWithStaticConfiguration.java From incubator-sentry with Apache License 2.0 | 5 votes |
private Partition makeMetastoreBasePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl) { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<String, String>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.setParameters(new HashMap<String, String>()); return part4; }
Example 8
Source File: HiveDifferencesTest.java From circus-train with Apache License 2.0 | 5 votes |
private static Partition newPartition(String databaseName, String tableName, String location) { Partition partition = new Partition(); partition.setDbName(databaseName); partition.setTableName(tableName); partition.setParameters(new HashMap<String, String>()); partition.setValues(Arrays.asList("01")); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(location); partition.setSd(sd); return partition; }
Example 9
Source File: HiveDifferencesIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition( String datbase, String table, StorageDescriptor tableStorageDescriptor, List<String> values, File location, String sourceTable, String sourceLocation, boolean addChecksum) { Partition partition = new Partition(); partition.setDbName(datbase); partition.setTableName(table); partition.setValues(values); partition.setSd(new StorageDescriptor(tableStorageDescriptor)); partition.getSd().setLocation(location.toURI().toString()); partition.setParameters(new HashMap<String, String>()); if (sourceTable != null) { partition.getParameters().put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), sourceTable); } if (sourceLocation != null) { partition.getParameters().put(CircusTrainTableParameter.SOURCE_LOCATION.parameterName(), sourceLocation); } if (addChecksum) { partition.getParameters().put(CircusTrainTableParameter.PARTITION_CHECKSUM.parameterName(), location.getName()); } return partition; }
Example 10
Source File: TestUtils.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newViewPartition(Table hiveView, List<String> values) { Partition partition = new Partition(); partition.setDbName(hiveView.getDbName()); partition.setTableName(hiveView.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveView.getSd())); partition.getSd().setLocation(null); return partition; }
Example 11
Source File: TestUtils.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) { Partition partition = new Partition(); partition.setDbName(hiveTable.getDbName()); partition.setTableName(hiveTable.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveTable.getSd())); partition.getSd().setLocation(location.toString()); return partition; }
Example 12
Source File: HiveEntityFactory.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newPartition(Table table, String... partitionValues) { Partition partition = new Partition(); partition.setTableName(table.getTableName()); partition.setDbName(table.getDbName()); partition.setValues(Arrays.asList(partitionValues)); partition.setSd(table.getSd()); partition.setParameters(new HashMap<String, String>()); return partition; }
Example 13
Source File: ReplicaTableFactoryTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void newReplicaPartitionStatisticsWithTransformation() throws MetaException { sourceTable.setPartitionKeys( Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null))); Partition replicaPartition = new Partition(sourcePartition); replicaPartition.setDbName(MAPPED_DB_NAME); replicaPartition.setTableName(MAPPED_TABLE_NAME); replicaPartition.setValues(Arrays.asList("A", "B")); ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(); ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(); List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2); ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME); columnStatisticsDesc .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues())); ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs); ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY, PartitionTransformation.IDENTITY, COLUMN_STATISTICS_TRANSFORMATION); ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition, sourcePartitionStatistics); assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is("new_db")); assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is("new_table")); assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("part=newPart")); assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2)); assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1)); assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2)); }
Example 14
Source File: HivePartitionManager.java From data-highway with Apache License 2.0 | 5 votes |
private Partition newHivePartition( String tableName, List<String> partitionValues, String location, Map<String, String> parameters) { Partition partition = new Partition(); partition.setDbName(databaseName); partition.setTableName(tableName); partition.setValues(partitionValues); parameters.forEach((key, value) -> partition.putToParameters(key, value)); partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION); partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant())); partition.setSd(AvroStorageDescriptorFactory.create(location)); return partition; }
Example 15
Source File: DestructiveReplicaTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition(String partitionValue, Path location1) { Partition partition = new Partition(); partition.setValues(Lists.newArrayList(partitionValue)); StorageDescriptor sd1 = new StorageDescriptor(); sd1.setLocation(location1.toString()); partition.setSd(sd1); Map<String, String> parameters = new HashMap<>(); parameters.put(REPLICATION_EVENT.parameterName(), EVENT_ID); partition.setParameters(parameters); return partition; }
Example 16
Source File: FilterToolIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition(StorageDescriptor tableStorageDescriptor, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(TABLE); partition.setValues(values); partition.setSd(new StorageDescriptor(tableStorageDescriptor)); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example 17
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 5 votes |
@Test public void dropPartition() throws Exception { HiveServer2Core server = setupServer(); String tableName = "my_table"; HiveMetaStoreClient client = server.getCore().newClient(); try { Table table = createPartitionedTable(DATABASE, tableName, server); Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(tableName); partition.setValues(Arrays.asList("1")); partition.setSd(new StorageDescriptor(table.getSd())); partition.getSd().setLocation( String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName)); client.add_partition(partition); try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl()); Statement statement = connection.createStatement()) { String dropPartitionHql = String.format("ALTER TABLE %s.%s DROP PARTITION (partcol=1)", DATABASE, tableName); statement.execute(dropPartitionHql); } List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1); assertThat(partitions.size(), is(0)); } finally { client.close(); } server.shutdown(); }
Example 18
Source File: HiveTableUtil.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a Hive partition instance. */ public static Partition createHivePartition(String dbName, String tableName, List<String> values, StorageDescriptor sd, Map<String, String> parameters) { Partition partition = new Partition(); partition.setDbName(dbName); partition.setTableName(tableName); partition.setValues(values); partition.setParameters(parameters); partition.setSd(sd); int currentTime = (int) (System.currentTimeMillis() / 1000); partition.setCreateTime(currentTime); partition.setLastAccessTime(currentTime); return partition; }
Example 19
Source File: HiveTableMetaStoreFactory.java From flink with Apache License 2.0 | 5 votes |
private void createPartition(LinkedHashMap<String, String> partSpec, Path path) throws Exception { StorageDescriptor newSd = new StorageDescriptor(sd); newSd.setLocation(path.toString()); Partition partition = HiveTableUtil.createHivePartition(database, tableName, new ArrayList<>(partSpec.values()), newSd, new HashMap<>()); partition.setValues(new ArrayList<>(partSpec.values())); client.add_partition(partition); }
Example 20
Source File: HiveConvertersImpl.java From metacat with Apache License 2.0 | 4 votes |
/** * {@inheritDoc} */ @Override public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) { final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null; String tableName = null; if (name != null) { if (name.getPartitionName() != null) { // // Unescape the partition name to get the right partition values. // Partition name always are escaped where as the parition values are not. // values = getPartValsFromName(tableDto, name.getPartitionName()); } if (name.getDatabaseName() != null) { databaseName = name.getDatabaseName(); } if (name.getTableName() != null) { tableName = name.getTableName(); } } result.setValues(values); result.setDbName(databaseName); result.setTableName(tableName); Map<String, String> metadata = partitionDto.getMetadata(); if (metadata == null) { metadata = Maps.newHashMap(); } result.setParameters(metadata); result.setSd(fromStorageDto(partitionDto.getSerde(), tableName)); final StorageDescriptor sd = result.getSd(); if (tableDto != null) { if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty( sd.getSerdeInfo().getSerializationLib())) { sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib()); } final List<FieldDto> fields = tableDto.getFields(); if (fields == null) { sd.setCols(Collections.emptyList()); } else { sd.setCols(fields.stream() .filter(field -> !field.isPartition_key()) .map(this::metacatToHiveField) .collect(Collectors.toList())); } } final AuditDto auditDto = partitionDto.getAudit(); if (auditDto != null) { if (auditDto.getCreatedDate() != null) { result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate())); } if (auditDto.getLastModifiedDate() != null) { result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate())); } } return result; }