Java Code Examples for org.apache.hadoop.hive.metastore.api.Table

The following examples show how to use org.apache.hadoop.hive.metastore.api.Table. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example 2
Source Project: metacat   Source File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_fields_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_fields_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);

            if (table == null || table.getSd() == null || table.getSd().getCols() == null) {
                throw new MetaException("Unable to get fields for " + dbName + "." + tableName);
            }
            return table.getSd().getCols();
        });
}
 
Example 3
Source Project: metacat   Source File: PartitionUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the partition values from the partition name. This method also validates the partition keys to that
 * of the table.
 *
 * @param tableQName  table name
 * @param table       table
 * @param partName    partition name
 * @return list of partition values
 */
public static List<String> getPartValuesFromPartName(final QualifiedName tableQName, final Table table,
    final String partName) {
    if (Strings.isNullOrEmpty(partName)) {
        throw new InvalidMetaException(tableQName, partName, null);
    }
    final LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
    Warehouse.makeSpecFromName(partSpec, new Path(partName));
    final List<String> values = new ArrayList<>();
    for (FieldSchema field : table.getPartitionKeys()) {
        final String key = field.getName();
        final String val = partSpec.get(key);
        if (val == null) {
            throw new InvalidMetaException(tableQName, partName, null);
        }
        values.add(val);
    }
    return values;
}
 
Example 4
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception {
  StorageDescriptor partitionSd = new StorageDescriptor();
  if (StringUtils.isNotBlank(tbl.getSd().getLocation())) {
    partitionSd.setLocation(tbl.getSd().getLocation() + values);
  } else {
    partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1");
  }

  partitionSd.setSerdeInfo(
      new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy")));
  partitionSd.setCols(tbl.getPartitionKeys());
  Partition partition =
      new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>());
  partition.setCreateTime(createTime);
  return this.getLocalMetastoreClient().add_partition(partition);

}
 
Example 5
Source Project: waggle-dance   Source File: DatabaseMappingImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void transformOutboundGetTablesResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  Table table2 = new Table();
  table2.setDbName(DB_NAME);
  table2.setTableName(TABLE_NAME);
  table2.setViewExpandedText(VIEW_EXPANDED_TEXT);
  table2.setViewOriginalText(VIEW_ORIGINAL_TEXT);
  GetTablesResult result = new GetTablesResult();
  result.setTables(Arrays.asList(table, table2));
  GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult.getTables().size(), is(2));
  assertThat(transformedResult.getTables().get(0), is(sameInstance(result.getTables().get(0))));
  assertThat(transformedResult.getTables().get(0).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(0).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(0).getViewExpandedText(), nullValue());
  assertThat(transformedResult.getTables().get(0).getViewOriginalText(), nullValue());
  assertThat(transformedResult.getTables().get(1), is(sameInstance(result.getTables().get(1))));
  assertThat(transformedResult.getTables().get(1).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(1).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(1).getViewExpandedText(), is(VIEW_EXPANDED_TEXT_TRANSFORMED));
  assertThat(transformedResult.getTables().get(1).getViewOriginalText(), is(VIEW_ORIGINAL_TEXT_TRANSFORMED));
}
 
Example 6
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example 7
Source Project: iceberg   Source File: HiveTableOperations.java    License: Apache License 2.0 6 votes vote down vote up
private void setParameters(String newMetadataLocation, Table tbl) {
  Map<String, String> parameters = tbl.getParameters();

  if (parameters == null) {
    parameters = new HashMap<>();
  }

  parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ENGLISH));
  parameters.put(METADATA_LOCATION_PROP, newMetadataLocation);

  if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
    parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
  }

  tbl.setParameters(parameters);
}
 
Example 8
Source Project: dremio-oss   Source File: HiveClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    default:
      return null;
  }
}
 
Example 9
Source Project: circus-train   Source File: CopyPartitionsOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 
Example 10
Source Project: incubator-gobblin   Source File: HiveSchemaEvolutionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvolutionDisabledForNewTable() throws IOException {
  boolean isEvolutionEnabled = false;
  Optional<Table> destinationTableMeta = Optional.absent();

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, true, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 
Example 11
Source Project: HiveKudu-Handler   Source File: KuduStorageHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void rollbackCreateTable(Table tbl) throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if ( client.tableExists(tablename) && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table while rollback of create table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example 12
private String getFilePath(Table tbl) throws Exception {

        StorageDescriptor descTable = tbl.getSd();

        InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(descTable.getInputFormat(), jobConf);

        FileInputFormat.setInputPaths(jobConf, new Path(descTable.getLocation()));

        InputSplit[] splits;
        try {
            splits = fformat.getSplits(jobConf, 1);
        } catch (org.apache.hadoop.mapred.InvalidInputException e) {
            LOG.debug("getSplits failed on " + e.getMessage());
            throw new RuntimeException("Unable to get file path for table.");
        }

        for (InputSplit split : splits) {
            FileSplit fsp = (FileSplit) split;
            String[] hosts = fsp.getLocations();
            String filepath = fsp.getPath().toString();
            return filepath;
        }
        throw new RuntimeException("Unable to get file path for table.");
    }
 
Example 13
@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example 14
Source Project: circus-train   Source File: AlterTableService.java    License: Apache License 2.0 6 votes vote down vote up
public void alterTable(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws Exception {
  List<FieldSchema> oldColumns = oldTable.getSd().getCols();
  List<FieldSchema> newColumns = newTable.getSd().getCols();
  if (hasAnyChangedColumns(oldColumns, newColumns)) {
    LOG
        .info("Found columns that have changed type, attempting to recreate target table with the new columns."
            + "Old columns: {}, new columns: {}", oldColumns, newColumns);
    Table tempTable = new Table(newTable);
    String tempName = newTable.getTableName() + "_temp";
    tempTable.setTableName(tempName);
    try {
      client.createTable(tempTable);
      copyPartitionsOperation.execute(client, newTable, tempTable);
      renameTableOperation.execute(client, tempTable, newTable);
    } finally {
      dropTableService.dropTable(client, tempTable.getDbName(), tempName);
    }
  } else {
    client.alter_table(newTable.getDbName(), newTable.getTableName(), newTable);
  }
}
 
Example 15
Source Project: iceberg   Source File: HiveTablesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreate() throws TException {
  // Table should be created in hive metastore
  final Table table = metastoreClient.getTable(DB_NAME, TABLE_NAME);

  // check parameters are in expected state
  final Map<String, String> parameters = table.getParameters();
  Assert.assertNotNull(parameters);
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP)));
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getTableType()));

  // Ensure the table is pointing to empty location
  Assert.assertEquals(getTableLocation(TABLE_NAME) , table.getSd().getLocation());

  // Ensure it is stored as unpartitioned table in hive.
  Assert.assertEquals(0 , table.getPartitionKeysSize());

  // Only 1 snapshotFile Should exist and no manifests should exist
  Assert.assertEquals(1, metadataVersionFiles(TABLE_NAME).size());
  Assert.assertEquals(0, manifestFiles(TABLE_NAME).size());

  final com.netflix.iceberg.Table icebergTable = new HiveTables(hiveConf).load(DB_NAME, TABLE_NAME);
  // Iceberg schema should match the loaded table
  Assert.assertEquals(schema.asStruct(), icebergTable.schema().asStruct());
}
 
Example 16
Source Project: presto   Source File: InMemoryThriftMetastore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void dropTable(HiveIdentity identity, String databaseName, String tableName, boolean deleteData)
{
    List<String> locations = listAllDataPaths(identity, this, databaseName, tableName);

    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Table table = relations.remove(schemaTableName);
    if (table == null) {
        throw new TableNotFoundException(schemaTableName);
    }
    views.remove(schemaTableName);
    partitions.keySet().removeIf(partitionName -> partitionName.matches(databaseName, tableName));

    // remove data
    if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
        for (String location : locations) {
            if (location != null) {
                File directory = new File(new Path(location).toUri());
                checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory");
                deleteDirectory(directory);
            }
        }
    }
}
 
Example 17
@Test
public void newSourcePartition() throws Exception {
  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  File sourcePartition2Location = createPartitionData("part=2", sourceTableUri,
      Arrays.asList("5\troberto", "6\tpedro"));
  Partition sourcePartition2 = newPartition(DATABASE, SOURCE_TABLE, sourceTable.getSd(), Arrays.asList("2"),
      sourcePartition2Location, null, null, false);
  catalog.client().add_partition(sourcePartition2);

  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);

  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();
  verify(diffListener, never()).onChangedTable(anyList());
  verify(diffListener, times(1))
      .onNewPartition("part=2", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=2"));
  verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList());
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}
 
Example 18
Source Project: circus-train   Source File: Source.java    License: Apache License 2.0 6 votes vote down vote up
public SourceLocationManager getLocationManager(
    Table table,
    List<Partition> partitions,
    String eventId,
    Map<String, Object> copierOptions)
  throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId,
      table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener);
  boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions,
      CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false);
  if (ignoreMissingFolder) {
    return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf());
  }
  return hdfsSnapshotLocationManager;
}
 
Example 19
Source Project: HiveKudu-Handler   Source File: KuduStorageHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void commitDropTable(Table tbl, boolean deleteData)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if (deleteData && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example 20
public static ConvertibleHiveDataset createTestConvertibleDataset(Config config)
    throws URISyntaxException {
  Table table = getTestTable("db1", "tb1");
  FileSystem mockFs = Mockito.mock(FileSystem.class);
  when(mockFs.getUri()).thenReturn(new URI("test"));
  ConvertibleHiveDataset cd =
      new ConvertibleHiveDataset(mockFs, Mockito.mock(HiveMetastoreClientPool.class), new org.apache.hadoop.hive.ql.metadata.Table(
          table), new Properties(), config);
  return cd;
}
 
Example 21
Source Project: incubator-gobblin   Source File: HiveSchemaEvolutionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEvolutionDisabledForExistingTable() throws IOException {
  boolean isEvolutionEnabled = false;
  boolean casePreserved = true;
  Optional<Table> destinationTableMeta = createEvolvedDestinationTable(schemaName, "default", "", true);

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, casePreserved, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 
Example 22
@Override
public void preCreateTable(Table tbl) throws MetaException {

    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    if (isExternal) {
        Log.info("Creating External table for Splice...");
    }

    String inputTableName = tbl.getParameters().get(MRConstants.SPLICE_TABLE_NAME);
    if (inputTableName == null)
        throw new MetaException("Wrong param, you are missing " +
        		MRConstants.SPLICE_TABLE_NAME + " ? ");

    // We can choose to support user define column mapping.
    // But currently I don't think it is necessary
    // We map all columns from Splice Table to Hive Table.
    String connStr = tbl.getParameters().get(MRConstants.SPLICE_JDBC_STR);
    if (connStr == null)
        throw new MetaException("Wrong param, did you mean " +
        		MRConstants.SPLICE_JDBC_STR + " ? ");
    if (sqlUtil == null)
        sqlUtil = SMSQLUtil.getInstance(connStr);
    if (inputTableName != null) {
        inputTableName = inputTableName.trim();
        checkTableExists(inputTableName);
    }
}
 
Example 23
@Test
public void unpartitionedTableReplicateAvroSchema() throws Exception {
  helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));
  LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));

  java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test");
  Files.write(sourceAvroSchemaPath, AVRO_SCHEMA_CONTENT.getBytes());
  String avroSchemaUrl = sourceAvroSchemaPath.toString();

  Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE);
  sourceTable.putToParameters("avro.schema.url", avroSchemaUrl);
  sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable);

  exit.expectSystemExitWithStatus(0);
  File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml");
  CircusTrainRunner runner = CircusTrainRunner
      .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation)
      .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(),
          sourceCatalog.driverClassName())
      .replicaMetaStore(replicaCatalog.getThriftConnectionUri())
      .build();

  exit.checkAssertionAfterwards(new Assertion() {
    @Override
    public void checkAssertion() throws Exception {
      Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE);
      String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/";
      String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
      assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl));
      Path copiedSchema = new Path(transformedAvroUrl);
      FileSystem fs = FileSystem.get(replicaCatalog.conf());
      assertTrue(fs.exists(copiedSchema));
      String content = new String(Files.readAllBytes(java.nio.file.Paths.get(copiedSchema.toUri())));
      assertThat(content, is(AVRO_SCHEMA_CONTENT));
    }
  });

  runner.run(config.getAbsolutePath());
}
 
Example 24
Source Project: Kylin   Source File: HiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
 * 
 * Get basic stats of table
 * @param table
 *          - table
 * @param statType
 *          - type of stats
 * @return value of stats
 */
public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
    Map<String, String> params = table.getParameters();
    long result = 0;

    if (params != null) {
        try {
            result = Long.parseLong(params.get(statType));
        } catch (NumberFormatException e) {
            result = 0;
        }
    }
    return result;
}
 
Example 25
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private String getPartitionName(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, Table hiveTable) throws PartitionSpecInvalidException {
	List<String> partitionCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partitionVals = getOrderedFullPartitionValues(partitionSpec, partitionCols, tablePath);
	List<String> partKVs = new ArrayList<>();
	for (int i = 0; i < partitionCols.size(); i++) {
		partKVs.add(partitionCols.get(i) + "=" + partitionVals.get(i));
	}
	return String.join("/", partKVs);
}
 
Example 26
Source Project: circus-train   Source File: HiveDifferences.java    License: Apache License 2.0 5 votes vote down vote up
private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 
Example 27
Source Project: waggle-dance   Source File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void alter_table_with_cascade() throws TException {
  Table table = new Table();
  handler.alter_table_with_cascade(DB_P, "table", table, true);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).alter_table_with_cascade(DB_P, "table", table, true);
}
 
Example 28
private void addColumnsToReplica(Table replicaTable) throws Exception {
  List<FieldSchema> columns = replicaTable.getSd().getCols();
  columns.add(new FieldSchema("age", "int", ""));
  columns.add(new FieldSchema("colour", "string", ""));
  replicaTable.getSd().setCols(columns);
  replicaCatalog.client().alter_table(replicaTable.getDbName(), replicaTable.getTableName(), replicaTable);
}
 
Example 29
Source Project: flink   Source File: HiveStatsUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive table.
 */
public static ColumnStatistics createTableColumnStats(
		Table hiveTable,
		Map<String, CatalogColumnStatisticsDataBase> colStats) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(true, hiveTable.getDbName(), hiveTable.getTableName());
	return createHiveColumnStatistics(colStats, hiveTable.getSd(), desc);
}
 
Example 30
Source Project: incubator-gobblin   Source File: HiveConvertExtractor.java    License: Apache License 2.0 5 votes vote down vote up
public HiveConvertExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException {
  super(state);

  if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
    log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY));
    return;
  }

  if (!(this.hiveDataset instanceof ConvertibleHiveDataset)) {
    throw new IllegalStateException("HiveConvertExtractor is only compatible with ConvertibleHiveDataset");
  }

  ConvertibleHiveDataset convertibleHiveDataset = (ConvertibleHiveDataset) this.hiveDataset;

  try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) {
    Table table = client.get().getTable(this.dbName, this.tableName);

    SchemaAwareHiveTable schemaAwareHiveTable = new SchemaAwareHiveTable(table, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getTableSchemaUrl(), fs));

    SchemaAwareHivePartition schemaAwareHivePartition = null;

    if (this.hiveWorkUnit.getPartitionName().isPresent() && this.hiveWorkUnit.getPartitionSchemaUrl().isPresent()) {

      Partition partition = client.get().getPartition(this.dbName, this.tableName, this.hiveWorkUnit.getPartitionName().get());
      schemaAwareHivePartition =
          new SchemaAwareHivePartition(table, partition, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getPartitionSchemaUrl().get(), fs));
    }

    QueryBasedHiveConversionEntity entity =
        new QueryBasedHiveConversionEntity(convertibleHiveDataset, schemaAwareHiveTable, Optional.fromNullable(schemaAwareHivePartition));
    this.conversionEntities.add(entity);
  }

}