org.apache.hadoop.hive.metastore.api.Table Java Examples
The following examples show how to use
org.apache.hadoop.hive.metastore.api.Table.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduStorageHandler.java From HiveKudu-Handler with Apache License 2.0 | 6 votes |
@Override public void commitDropTable(Table tbl, boolean deleteData) throws MetaException { KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME)); String tablename = getKuduTableName(tbl); boolean isExternal = MetaStoreUtils.isExternalTable(tbl); try { if (deleteData && !isExternal) { client.deleteTable(tablename); } } catch (Exception ioe) { throw new MetaException("Error dropping table:" +tablename); } finally { try { client.shutdown(); } catch (Exception e) { e.printStackTrace(); } } }
Example #2
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 6 votes |
/** * {@inheritDoc} */ @Override public List<FieldSchema> get_fields_with_environment_context( final String dbName, final String tableName, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("get_fields_with_environment_context", new Object[]{dbName, tableName, environmentContext}, () -> { final Table table = get_table(dbName, tableName); if (table == null || table.getSd() == null || table.getSd().getCols() == null) { throw new MetaException("Unable to get fields for " + dbName + "." + tableName); } return table.getSd().getCols(); }); }
Example #3
Source File: PartitionUtil.java From metacat with Apache License 2.0 | 6 votes |
/** * Retrieves the partition values from the partition name. This method also validates the partition keys to that * of the table. * * @param tableQName table name * @param table table * @param partName partition name * @return list of partition values */ public static List<String> getPartValuesFromPartName(final QualifiedName tableQName, final Table table, final String partName) { if (Strings.isNullOrEmpty(partName)) { throw new InvalidMetaException(tableQName, partName, null); } final LinkedHashMap<String, String> partSpec = new LinkedHashMap<>(); Warehouse.makeSpecFromName(partSpec, new Path(partName)); final List<String> values = new ArrayList<>(); for (FieldSchema field : table.getPartitionKeys()) { final String key = field.getName(); final String val = partSpec.get(key); if (val == null) { throw new InvalidMetaException(tableQName, partName, null); } values.add(val); } return values; }
Example #4
Source File: LocalHiveMetastoreTestUtils.java From incubator-gobblin with Apache License 2.0 | 6 votes |
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }
Example #5
Source File: DatabaseMappingImplTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void transformOutboundGetTablesResult() throws Exception { Table table = new Table(); table.setDbName(DB_NAME); table.setTableName(TABLE_NAME); Table table2 = new Table(); table2.setDbName(DB_NAME); table2.setTableName(TABLE_NAME); table2.setViewExpandedText(VIEW_EXPANDED_TEXT); table2.setViewOriginalText(VIEW_ORIGINAL_TEXT); GetTablesResult result = new GetTablesResult(); result.setTables(Arrays.asList(table, table2)); GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result); assertThat(transformedResult, is(sameInstance(result))); assertThat(transformedResult.getTables().size(), is(2)); assertThat(transformedResult.getTables().get(0), is(sameInstance(result.getTables().get(0)))); assertThat(transformedResult.getTables().get(0).getDbName(), is(OUT_DB_NAME)); assertThat(transformedResult.getTables().get(0).getTableName(), is(TABLE_NAME)); assertThat(transformedResult.getTables().get(0).getViewExpandedText(), nullValue()); assertThat(transformedResult.getTables().get(0).getViewOriginalText(), nullValue()); assertThat(transformedResult.getTables().get(1), is(sameInstance(result.getTables().get(1)))); assertThat(transformedResult.getTables().get(1).getDbName(), is(OUT_DB_NAME)); assertThat(transformedResult.getTables().get(1).getTableName(), is(TABLE_NAME)); assertThat(transformedResult.getTables().get(1).getViewExpandedText(), is(VIEW_EXPANDED_TEXT_TRANSFORMED)); assertThat(transformedResult.getTables().get(1).getViewOriginalText(), is(VIEW_ORIGINAL_TEXT_TRANSFORMED)); }
Example #6
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception { Table table = new Table(); table.setDbName(DATABASE); table.setTableName(tableName); table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null))); table.setSd(new StorageDescriptor()); table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null))); table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); table.getSd().setSerdeInfo(new SerDeInfo()); table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); HiveMetaStoreClient client = server.getCore().newClient(); client.createTable(table); client.close(); return table; }
Example #7
Source File: HiveTableOperations.java From iceberg with Apache License 2.0 | 6 votes |
private void setParameters(String newMetadataLocation, Table tbl) { Map<String, String> parameters = tbl.getParameters(); if (parameters == null) { parameters = new HashMap<>(); } parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ENGLISH)); parameters.put(METADATA_LOCATION_PROP, newMetadataLocation); if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) { parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation()); } tbl.setParameters(parameters); }
Example #8
Source File: HiveClientImpl.java From dremio-oss with Apache License 2.0 | 6 votes |
@Override public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{ Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors); if(table == null){ return null; } TableType type = TableType.valueOf(table.getTableType()); switch (type) { case EXTERNAL_TABLE: case MANAGED_TABLE: return table; case VIRTUAL_VIEW: throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER); default: return null; } }
Example #9
Source File: CopyPartitionsOperation.java From circus-train with Apache License 2.0 | 6 votes |
/** * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable */ public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException { int count = 0; String databaseName = newTable.getDbName(); String tableName = newTable.getTableName(); PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize); while (partitionIterator.hasNext()) { List<Partition> batch = new ArrayList<>(); for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) { Partition partition = partitionIterator.next(); count++; Partition copy = new Partition(partition); copy.setDbName(databaseName); copy.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(partition.getSd()); sd.setCols(newTable.getSd().getCols()); copy.setSd(sd); batch.add(copy); } LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName); client.add_partitions(batch); } LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName); }
Example #10
Source File: HiveSchemaEvolutionTest.java From incubator-gobblin with Apache License 2.0 | 6 votes |
@Test public void testEvolutionDisabledForNewTable() throws IOException { boolean isEvolutionEnabled = false; Optional<Table> destinationTableMeta = Optional.absent(); String ddl = HiveAvroORCQueryGenerator .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName, Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(), Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), null, isEvolutionEnabled, true, destinationTableMeta, new HashMap<String, String>()); Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir, "source_schema_evolution_enabled.ddl"), "Generated DDL did not match expected for evolution disabled"); String dml = HiveAvroORCQueryGenerator .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(), Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(), Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit); Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir, "source_schema_evolution_enabled.dml"), "Generated DML did not match expected for evolution disabled"); }
Example #11
Source File: MultipleHiveFragmentsPerFileFragmenter.java From pxf with Apache License 2.0 | 6 votes |
private String getFilePath(Table tbl) throws Exception { StorageDescriptor descTable = tbl.getSd(); InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(descTable.getInputFormat(), jobConf); FileInputFormat.setInputPaths(jobConf, new Path(descTable.getLocation())); InputSplit[] splits; try { splits = fformat.getSplits(jobConf, 1); } catch (org.apache.hadoop.mapred.InvalidInputException e) { LOG.debug("getSplits failed on " + e.getMessage()); throw new RuntimeException("Unable to get file path for table."); } for (InputSplit split : splits) { FileSplit fsp = (FileSplit) split; String[] hosts = fsp.getLocations(); String filepath = fsp.getPath().toString(); return filepath; } throw new RuntimeException("Unable to get file path for table."); }
Example #12
Source File: DynamoDBStorageHandlerTest.java From emr-dynamodb-connector with Apache License 2.0 | 6 votes |
@Test public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException { TableDescription description = getHashRangeTable(); Table table = new Table(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," + "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap"); table.setParameters(parameters); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = Lists.newArrayList(); cols.add(new FieldSchema("col1", "string", "")); cols.add(new FieldSchema("hashMap", "map<string,string>", "")); sd.setCols(cols); table.setSd(sd); exceptionRule.expect(MetaException.class); exceptionRule.expectMessage("Could not find column(s) for column mapping(s): "); exceptionRule.expectMessage("col2:dynamo_col2#"); exceptionRule.expectMessage("hashkey:hashKey"); storageHandler.checkTableSchemaMapping(description, table); }
Example #13
Source File: AlterTableService.java From circus-train with Apache License 2.0 | 6 votes |
public void alterTable(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws Exception { List<FieldSchema> oldColumns = oldTable.getSd().getCols(); List<FieldSchema> newColumns = newTable.getSd().getCols(); if (hasAnyChangedColumns(oldColumns, newColumns)) { LOG .info("Found columns that have changed type, attempting to recreate target table with the new columns." + "Old columns: {}, new columns: {}", oldColumns, newColumns); Table tempTable = new Table(newTable); String tempName = newTable.getTableName() + "_temp"; tempTable.setTableName(tempName); try { client.createTable(tempTable); copyPartitionsOperation.execute(client, newTable, tempTable); renameTableOperation.execute(client, tempTable, newTable); } finally { dropTableService.dropTable(client, tempTable.getDbName(), tempName); } } else { client.alter_table(newTable.getDbName(), newTable.getTableName(), newTable); } }
Example #14
Source File: HiveTablesTest.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testCreate() throws TException { // Table should be created in hive metastore final Table table = metastoreClient.getTable(DB_NAME, TABLE_NAME); // check parameters are in expected state final Map<String, String> parameters = table.getParameters(); Assert.assertNotNull(parameters); Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP))); Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getTableType())); // Ensure the table is pointing to empty location Assert.assertEquals(getTableLocation(TABLE_NAME) , table.getSd().getLocation()); // Ensure it is stored as unpartitioned table in hive. Assert.assertEquals(0 , table.getPartitionKeysSize()); // Only 1 snapshotFile Should exist and no manifests should exist Assert.assertEquals(1, metadataVersionFiles(TABLE_NAME).size()); Assert.assertEquals(0, manifestFiles(TABLE_NAME).size()); final com.netflix.iceberg.Table icebergTable = new HiveTables(hiveConf).load(DB_NAME, TABLE_NAME); // Iceberg schema should match the loaded table Assert.assertEquals(schema.asStruct(), icebergTable.schema().asStruct()); }
Example #15
Source File: InMemoryThriftMetastore.java From presto with Apache License 2.0 | 6 votes |
@Override public synchronized void dropTable(HiveIdentity identity, String databaseName, String tableName, boolean deleteData) { List<String> locations = listAllDataPaths(identity, this, databaseName, tableName); SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName); Table table = relations.remove(schemaTableName); if (table == null) { throw new TableNotFoundException(schemaTableName); } views.remove(schemaTableName); partitions.keySet().removeIf(partitionName -> partitionName.matches(databaseName, tableName)); // remove data if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) { for (String location : locations) { if (location != null) { File directory = new File(new Path(location).toUri()); checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory"); deleteDirectory(directory); } } } }
Example #16
Source File: HiveDifferencesIntegrationTest.java From circus-train with Apache License 2.0 | 6 votes |
@Test public void newSourcePartition() throws Exception { Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE); File sourcePartition2Location = createPartitionData("part=2", sourceTableUri, Arrays.asList("5\troberto", "6\tpedro")); Partition sourcePartition2 = newPartition(DATABASE, SOURCE_TABLE, sourceTable.getSd(), Arrays.asList("2"), sourcePartition2Location, null, null, false); catalog.client().add_partition(sourcePartition2); Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE); HiveDifferences .builder(diffListener) .comparatorRegistry(comparatorRegistry) .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE)) .replica(Optional.of(replicaTable), Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE))) .checksumFunction(checksumFunction) .build() .run(); verify(diffListener, never()).onChangedTable(anyList()); verify(diffListener, times(1)) .onNewPartition("part=2", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=2")); verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList()); verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class)); }
Example #17
Source File: KuduStorageHandler.java From HiveKudu-Handler with Apache License 2.0 | 6 votes |
@Override public void rollbackCreateTable(Table tbl) throws MetaException { KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME)); String tablename = getKuduTableName(tbl); boolean isExternal = MetaStoreUtils.isExternalTable(tbl); try { if ( client.tableExists(tablename) && !isExternal) { client.deleteTable(tablename); } } catch (Exception ioe) { throw new MetaException("Error dropping table while rollback of create table:" +tablename); } finally { try { client.shutdown(); } catch (Exception e) { e.printStackTrace(); } } }
Example #18
Source File: Source.java From circus-train with Apache License 2.0 | 6 votes |
public SourceLocationManager getLocationManager( Table table, List<Partition> partitions, String eventId, Map<String, Object> copierOptions) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions, CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false); if (ignoreMissingFolder) { return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf()); } return hdfsSnapshotLocationManager; }
Example #19
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void typical() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY) .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); // Local table Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE); Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); assertThat(waggledLocalTable, is(localTable)); // Remote table String waggledRemoteDbName = REMOTE_DATABASE; assertTypicalRemoteTable(proxy, waggledRemoteDbName); }
Example #20
Source File: TableTransformationTest.java From circus-train with Apache License 2.0 | 5 votes |
@Before public void init() { table = new Table(); table.setDbName("database"); table.setTableName("table"); table.setTableType("type"); Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>(); userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo())); PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet(); privileges.setUserPrivileges(userPrivileges); table.setPrivileges(privileges); StorageDescriptor storageDescriptor = new StorageDescriptor(); storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null))); storageDescriptor.setInputFormat("input_format"); storageDescriptor.setOutputFormat("output_format"); storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>())); storageDescriptor.setSkewedInfo(new SkewedInfo()); storageDescriptor.setParameters(new HashMap<String, String>()); storageDescriptor.setLocation("database/table/"); table.setSd(storageDescriptor); Map<String, String> parameters = new HashMap<>(); parameters.put("com.company.parameter", "abc"); table.setParameters(parameters); }
Example #21
Source File: CatalogThriftHiveMetastore.java From metacat with Apache License 2.0 | 5 votes |
/** * {@inheritDoc} */ @Override public List<FieldSchema> get_schema_with_environment_context( final String dbName, final String tableName, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("get_schema_with_environment_context", new Object[]{dbName, tableName, environmentContext}, () -> { final Table table = get_table(dbName, tableName); List<FieldSchema> partitionKeys = Collections.emptyList(); List<FieldSchema> columns = Collections.emptyList(); if (table != null && table.getSd() != null && table.getSd().getCols() != null) { columns = table.getSd().getCols(); } if (table != null && table.getPartitionKeys() != null) { partitionKeys = table.getPartitionKeys(); } if (partitionKeys.isEmpty() && columns.isEmpty()) { throw new MetaException( "Table does not have any partition keys or cols: " + dbName + "." + tableName); } final List<FieldSchema> result = Lists.newArrayListWithCapacity(partitionKeys.size() + columns.size()); result.addAll(columns); result.addAll(partitionKeys); return result; }); }
Example #22
Source File: HiveMetadataFetcherTest.java From pxf with Apache License 2.0 | 5 votes |
@Test public void getTableMetadata() throws Exception { fetcher = new HiveMetadataFetcher(context, mockConfigurationFactory, fakeHiveClientWrapper); String tableName = "cause"; // mock hive table returned from hive client List<FieldSchema> fields = new ArrayList<>(); fields.add(new FieldSchema("field1", "string", null)); fields.add(new FieldSchema("field2", "int", null)); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(fields); sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); Table hiveTable = new Table(); hiveTable.setTableType("MANAGED_TABLE"); hiveTable.setSd(sd); hiveTable.setPartitionKeys(new ArrayList<>()); when(mockHiveClient.getTable("default", tableName)).thenReturn(hiveTable); // Get metadata metadataList = fetcher.getMetadata(tableName); Metadata metadata = metadataList.get(0); assertEquals("default.cause", metadata.getItem().toString()); List<Metadata.Field> resultFields = metadata.getFields(); assertNotNull(resultFields); assertEquals(2, resultFields.size()); Metadata.Field field = resultFields.get(0); assertEquals("field1", field.getName()); assertEquals("text", field.getType().getTypeName()); // converted type field = resultFields.get(1); assertEquals("field2", field.getName()); assertEquals("int4", field.getType().getTypeName()); }
Example #23
Source File: HiveDifferences.java From circus-train with Apache License 2.0 | 5 votes |
private static String partitionName(Table table, Partition partition) { try { return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues()); } catch (MetaException e) { throw new CircusTrainException("Unable to build partition name for partition values " + partition.getValues() + " of table " + Warehouse.getQualifiedName(table), e); } }
Example #24
Source File: ReplicaTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void alteringExistingUnpartitionedReplicaTableSucceeds() throws Exception { existingReplicaTable.getParameters().put(REPLICATION_EVENT.parameterName(), "previousEventId"); replica.updateMetadata(EVENT_ID, tableAndStatistics, DB_NAME, TABLE_NAME, mockReplicaLocationManager); verify(alterTableService).alterTable(eq(mockMetaStoreClient), eq(existingReplicaTable), any(Table.class)); verify(mockMetaStoreClient).updateTableColumnStatistics(columnStatistics); verify(mockReplicaLocationManager, never()).addCleanUpLocation(anyString(), any(Path.class)); }
Example #25
Source File: ReplicaTableFactoryTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void newTable() { TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME, REPLICA_DATA_DESTINATION, FULL); Table replica = replicaAndStats.getTable(); assertThat(replica.getDbName(), is(sourceTable.getDbName())); assertThat(replica.getTableName(), is(sourceTable.getTableName())); assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT)); assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT)); assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString())); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"), is(SOURCE_META_STORE_URIS)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID)); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue()))); assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name())); assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true")); assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true")); assertThat(replica.getParameters().get("STATS_GENERATED"), is("true")); assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1")); assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name())); assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE")); assertTrue(MetaStoreUtils.isExternalTable(replica)); assertThat(replicaAndStats.getStatistics(), is(nullValue())); }
Example #26
Source File: CircusTrainHdfsHdfsIntegrationTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void unpartitionedTableReplicateAvroSchema() throws Exception { helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test"); Files.write(sourceAvroSchemaPath, AVRO_SCHEMA_CONTENT.getBytes()); String avroSchemaUrl = sourceAvroSchemaPath.toString(); Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE); sourceTable.putToParameters("avro.schema.url", avroSchemaUrl); sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE); String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/"; String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url"); assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl)); Path copiedSchema = new Path(transformedAvroUrl); FileSystem fs = FileSystem.get(replicaCatalog.conf()); assertTrue(fs.exists(copiedSchema)); String content = new String(Files.readAllBytes(java.nio.file.Paths.get(copiedSchema.toUri()))); assertThat(content, is(AVRO_SCHEMA_CONTENT)); } }); runner.run(config.getAbsolutePath()); }
Example #27
Source File: TableParametersTransformation.java From circus-train with Apache License 2.0 | 5 votes |
private Map<String, String> mergeTableParameters(Map<String, String> tableParameters, Table table) { Map<String, String> parameters; if (table.getParameters() != null) { parameters = new LinkedHashMap<>(table.getParameters()); } else { parameters = new LinkedHashMap<>(); } parameters.putAll(tableParameters); return parameters; }
Example #28
Source File: Source.java From circus-train with Apache License 2.0 | 5 votes |
public SourceLocationManager getLocationManager(Table table, String eventId) throws IOException { if (MetaStoreUtils.isView(table)) { return new ViewLocationManager(); } return new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, snapshotsDisabled, sourceTableLocation, sourceCatalogListener); }
Example #29
Source File: MetacatHMSHandler.java From metacat with Apache License 2.0 | 5 votes |
@SuppressFBWarnings private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, final boolean madeDir) throws MetaException { // set create time final long time = System.currentTimeMillis() / 1000; part.setCreateTime((int) time); if (part.getParameters() == null || part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) { part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); } // Inherit table properties into partition properties. final Map<String, String> tblParams = tbl.getParameters(); final String inheritProps = getHiveConf().getVar(HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim(); // Default value is empty string in which case no properties will be inherited. // * implies all properties needs to be inherited Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(","))); if (inheritKeys.contains("*")) { inheritKeys = tblParams.keySet(); } for (String key : inheritKeys) { final String paramVal = tblParams.get(key); if (null != paramVal) { // add the property only if it exists in table properties part.putToParameters(key, paramVal); } } }
Example #30
Source File: HiveClient.java From Kylin with Apache License 2.0 | 5 votes |
/** * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility * * Get basic stats of table * @param table * - table * @param statType * - type of stats * @return value of stats */ public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) { Map<String, String> params = table.getParameters(); long result = 0; if (params != null) { try { result = Long.parseLong(params.get(statType)); } catch (NumberFormatException e) { result = 0; } } return result; }