Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setDbName()
The following examples show how to use
org.apache.hadoop.hive.metastore.api.Partition#setDbName() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CopyPartitionsOperation.java From circus-train with Apache License 2.0 | 6 votes |
/** * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable */ public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException { int count = 0; String databaseName = newTable.getDbName(); String tableName = newTable.getTableName(); PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize); while (partitionIterator.hasNext()) { List<Partition> batch = new ArrayList<>(); for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) { Partition partition = partitionIterator.next(); count++; Partition copy = new Partition(partition); copy.setDbName(databaseName); copy.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(partition.getSd()); sd.setCols(newTable.getSd().getCols()); copy.setSd(sd); batch.add(copy); } LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName); client.add_partitions(batch); } LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName); }
Example 2
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void add_partitions_req() throws TException { Partition newPartition1 = new Partition(); newPartition1.setDbName(DB_P); Partition newPartition2 = new Partition(); newPartition2.setDbName(DB_P); List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2); AddPartitionsRequest request = new AddPartitionsRequest(); request.setDbName(DB_P); request.setParts(partitions); AddPartitionsRequest inbound = new AddPartitionsRequest(); AddPartitionsResult addPartitionResult = new AddPartitionsResult(); AddPartitionsResult outbound = new AddPartitionsResult(); when(primaryMapping.transformInboundAddPartitionsRequest(request)).thenReturn(inbound); when(primaryClient.add_partitions_req(inbound)).thenReturn(addPartitionResult); when(primaryMapping.transformOutboundAddPartitionsResult(addPartitionResult)).thenReturn(outbound); AddPartitionsResult result = handler.add_partitions_req(request); assertThat(result, is(outbound)); verify(primaryMapping, times(3)).checkWritePermissions(DB_P); }
Example 3
Source File: ReplicaTableFactoryTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void newReplicaPartitionStatistics() throws MetaException { sourceTable.setPartitionKeys( Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null))); Partition replicaPartition = new Partition(sourcePartition); replicaPartition.setDbName(MAPPED_DB_NAME); replicaPartition.setTableName(MAPPED_TABLE_NAME); replicaPartition.setValues(Arrays.asList("A", "B")); ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(); ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(); List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2); ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME); columnStatisticsDesc .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues())); ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs); ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition, sourcePartitionStatistics); assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME)); assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME)); assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B")); assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2)); assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1)); assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2)); }
Example 4
Source File: PartitionTransformationTest.java From circus-train with Apache License 2.0 | 5 votes |
@Before public void init() { partition = new Partition(); partition.setDbName("database"); partition.setTableName("table"); partition.setValues(ImmutableList.of("part")); Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>(); userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo())); PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet(); privileges.setUserPrivileges(userPrivileges); partition.setPrivileges(privileges); StorageDescriptor storageDescriptor = new StorageDescriptor(); storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null))); storageDescriptor.setInputFormat("input_format"); storageDescriptor.setOutputFormat("output_format"); storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>())); storageDescriptor.setSkewedInfo(new SkewedInfo()); storageDescriptor.setParameters(new HashMap<String, String>()); storageDescriptor.setLocation("database/table/part/"); partition.setSd(storageDescriptor); Map<String, String> parameters = new HashMap<>(); parameters.put("com.company.parameter", "abc"); partition.setParameters(parameters); }
Example 5
Source File: TestUtils.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newPartition(String database, String tableName, String partitionValue) { Partition partition = new Partition(); partition.setDbName(database); partition.setTableName(tableName); partition.setCreateTime(CREATE_TIME); partition.setValues(ImmutableList.of(partitionValue)); Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>(); userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo())); PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet(); privileges.setUserPrivileges(userPrivileges); partition.setPrivileges(privileges); StorageDescriptor storageDescriptor = new StorageDescriptor(); storageDescriptor.setCols(COLS); storageDescriptor.setInputFormat(INPUT_FORMAT); storageDescriptor.setOutputFormat(OUTPUT_FORMAT); storageDescriptor.setSerdeInfo(new SerDeInfo(SERDE_INFO_NAME, SERIALIZATION_LIB, new HashMap<String, String>())); storageDescriptor.setSkewedInfo(new SkewedInfo()); storageDescriptor.setParameters(new HashMap<String, String>()); storageDescriptor.setLocation(DATABASE + "/" + tableName + "/" + partitionValue + "/"); partition.setSd(storageDescriptor); Map<String, String> parameters = new HashMap<>(); parameters.put("com.company.parameter", "abc"); partition.setParameters(parameters); return partition; }
Example 6
Source File: TestUtils.java From waggle-dance with Apache License 2.0 | 5 votes |
static Partition newPartition(Table hiveTable, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(hiveTable.getDbName()); partition.setTableName(hiveTable.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveTable.getSd())); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example 7
Source File: TestHiveCleanService.java From Hue-Ctrip-DI with MIT License | 5 votes |
private void add_partition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws InvalidObjectException, AlreadyExistsException, MetaException, TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<String, String>()); part.setSd(table.getSd()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); }
Example 8
Source File: TestUtils.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newViewPartition(Table hiveView, List<String> values) { Partition partition = new Partition(); partition.setDbName(hiveView.getDbName()); partition.setTableName(hiveView.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveView.getSd())); partition.getSd().setLocation(null); return partition; }
Example 9
Source File: TestUtils.java From circus-train with Apache License 2.0 | 5 votes |
public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) { Partition partition = new Partition(); partition.setDbName(hiveTable.getDbName()); partition.setTableName(hiveTable.getTableName()); partition.setValues(values); partition.setSd(new StorageDescriptor(hiveTable.getSd())); partition.getSd().setLocation(location.toString()); return partition; }
Example 10
Source File: HivePartitionManager.java From data-highway with Apache License 2.0 | 5 votes |
private Partition newHivePartition( String tableName, List<String> partitionValues, String location, Map<String, String> parameters) { Partition partition = new Partition(); partition.setDbName(databaseName); partition.setTableName(tableName); partition.setValues(partitionValues); parameters.forEach((key, value) -> partition.putToParameters(key, value)); partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION); partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant())); partition.setSd(AvroStorageDescriptorFactory.create(location)); return partition; }
Example 11
Source File: DatabaseMappingImplTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Before public void setUp() { databaseMapping = new DatabaseMappingImpl(metastoreMapping, queryMapping); database = new Database(); database.setName(DB_NAME); partition = new Partition(); partition.setDbName(DB_NAME); partitions = Lists.newArrayList(partition); index = new Index(); index.setDbName(DB_NAME); hiveObjectRef = new HiveObjectRef(); hiveObjectRef.setDbName(DB_NAME); hiveObjectRef.setObjectType(HiveObjectType.DATABASE); hiveObjectRef.setObjectName(DB_NAME); hiveObjectPrivileges = new ArrayList<>(); HiveObjectPrivilege hiveObjectPrivilege = new HiveObjectPrivilege(); hiveObjectPrivilege.setHiveObject(hiveObjectRef); hiveObjectPrivileges.add(hiveObjectPrivilege); partitionSpec = new PartitionSpec(); partitionSpec.setDbName(DB_NAME); when(metastoreMapping.transformInboundDatabaseName(anyString())).thenReturn(IN_DB_NAME); when(metastoreMapping.transformOutboundDatabaseName(anyString())).thenReturn(OUT_DB_NAME); when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_EXPANDED_TEXT)) .thenReturn(VIEW_EXPANDED_TEXT_TRANSFORMED); when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_ORIGINAL_TEXT)) .thenReturn(VIEW_ORIGINAL_TEXT_TRANSFORMED); }
Example 12
Source File: ReplicaTableFactory.java From circus-train with Apache License 2.0 | 5 votes |
Partition newReplicaPartition( String eventId, Table sourceTable, Partition sourcePartition, String replicaDatabaseName, String replicaTableName, Path replicaPartitionLocation, ReplicationMode replicationMode) { Partition replica = partitionTransformation.transform(new Partition(sourcePartition)); replica.setDbName(replicaDatabaseName); replica.setTableName(replicaTableName); if (replica.getSd() != null) { replica.getSd().setLocation(toStringOrNull(replicaPartitionLocation)); } String sourcePartitionLocation = sourcePartition.getSd() == null ? "" : toStringOrEmpty(sourcePartition.getSd().getLocation()); // Statistic specific parameters replica.putToParameters(STATS_GENERATED_VIA_STATS_TASK, Boolean.TRUE.toString()); replica.putToParameters(STATS_GENERATED, Boolean.TRUE.toString()); replica.putToParameters(DO_NOT_UPDATE_STATS, Boolean.TRUE.toString()); // Replication specific parameters replica.putToParameters(LAST_REPLICATED.parameterName(), DateTime.now(DateTimeZone.UTC).toString()); replica.putToParameters(REPLICATION_EVENT.parameterName(), eventId); replica.putToParameters(SOURCE_LOCATION.parameterName(), sourcePartitionLocation); replica.putToParameters(SOURCE_TABLE.parameterName(), Warehouse.getQualifiedName(sourceTable)); replica.putToParameters(SOURCE_METASTORE.parameterName(), sourceMetaStoreUris); replica.putToParameters(REPLICATION_MODE.parameterName(), replicationMode.name()); return replica; }
Example 13
Source File: FilterToolIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition(StorageDescriptor tableStorageDescriptor, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(TABLE); partition.setValues(values); partition.setSd(new StorageDescriptor(tableStorageDescriptor)); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example 14
Source File: ComparisonToolIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
private Partition newPartition( String table, StorageDescriptor tableStorageDescriptor, List<String> values, File location) { Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(table); partition.setValues(values); partition.setSd(new StorageDescriptor(tableStorageDescriptor)); partition.getSd().setLocation(location.toURI().toString()); return partition; }
Example 15
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 5 votes |
@Test public void dropPartition() throws Exception { HiveServer2Core server = setupServer(); String tableName = "my_table"; HiveMetaStoreClient client = server.getCore().newClient(); try { Table table = createPartitionedTable(DATABASE, tableName, server); Partition partition = new Partition(); partition.setDbName(DATABASE); partition.setTableName(tableName); partition.setValues(Arrays.asList("1")); partition.setSd(new StorageDescriptor(table.getSd())); partition.getSd().setLocation( String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName)); client.add_partition(partition); try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl()); Statement statement = connection.createStatement()) { String dropPartitionHql = String.format("ALTER TABLE %s.%s DROP PARTITION (partcol=1)", DATABASE, tableName); statement.execute(dropPartitionHql); } List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1); assertThat(partitions.size(), is(0)); } finally { client.close(); } server.shutdown(); }
Example 16
Source File: FederatedHMSHandlerTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void alter_partitions() throws TException { Partition newPartition1 = new Partition(); newPartition1.setDbName(DB_P); Partition newPartition2 = new Partition(); newPartition2.setDbName(DB_P); List<Partition> inbound = Lists.newArrayList(new Partition()); List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2); when(primaryMapping.transformInboundPartitions(partitions)).thenReturn(inbound); handler.alter_partitions(DB_P, "table", partitions); verify(primaryMapping, times(3)).checkWritePermissions(DB_P); verify(primaryClient).alter_partitions(DB_P, "table", inbound); }
Example 17
Source File: AbstractMetastoreTestWithStaticConfiguration.java From incubator-sentry with Apache License 2.0 | 5 votes |
private Partition makeMetastoreBasePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl) { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<String, String>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.setParameters(new HashMap<String, String>()); return part4; }
Example 18
Source File: DatabaseMappingImpl.java From waggle-dance with Apache License 2.0 | 4 votes |
@Override public Partition transformOutboundPartition(Partition partition) { partition.setDbName(metaStoreMapping.transformOutboundDatabaseName(partition.getDbName())); return partition; }
Example 19
Source File: HiveConvertersImpl.java From metacat with Apache License 2.0 | 4 votes |
/** * {@inheritDoc} */ @Override public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) { final Partition result = new Partition(); final QualifiedName name = partitionDto.getName(); List<String> values = Lists.newArrayListWithCapacity(16); String databaseName = null; String tableName = null; if (name != null) { if (name.getPartitionName() != null) { // // Unescape the partition name to get the right partition values. // Partition name always are escaped where as the parition values are not. // values = getPartValsFromName(tableDto, name.getPartitionName()); } if (name.getDatabaseName() != null) { databaseName = name.getDatabaseName(); } if (name.getTableName() != null) { tableName = name.getTableName(); } } result.setValues(values); result.setDbName(databaseName); result.setTableName(tableName); Map<String, String> metadata = partitionDto.getMetadata(); if (metadata == null) { metadata = Maps.newHashMap(); } result.setParameters(metadata); result.setSd(fromStorageDto(partitionDto.getSerde(), tableName)); final StorageDescriptor sd = result.getSd(); if (tableDto != null) { if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty( sd.getSerdeInfo().getSerializationLib())) { sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib()); } final List<FieldDto> fields = tableDto.getFields(); if (fields == null) { sd.setCols(Collections.emptyList()); } else { sd.setCols(fields.stream() .filter(field -> !field.isPartition_key()) .map(this::metacatToHiveField) .collect(Collectors.toList())); } } final AuditDto auditDto = partitionDto.getAudit(); if (auditDto != null) { if (auditDto.getCreatedDate() != null) { result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate())); } if (auditDto.getLastModifiedDate() != null) { result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate())); } } return result; }
Example 20
Source File: DatabaseMappingImpl.java From waggle-dance with Apache License 2.0 | 4 votes |
@Override public Partition transformInboundPartition(Partition partition) { partition.setDbName(metaStoreMapping.transformInboundDatabaseName(partition.getDbName())); return partition; }