Java Code Examples for org.apache.hadoop.hive.metastore.api.Table

The following examples show how to use org.apache.hadoop.hive.metastore.api.Table. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: waggle-dance   Author: HotelsDotCom   File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example #2
Source Project: metacat   Author: Netflix   File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_fields_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_fields_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);

            if (table == null || table.getSd() == null || table.getSd().getCols() == null) {
                throw new MetaException("Unable to get fields for " + dbName + "." + tableName);
            }
            return table.getSd().getCols();
        });
}
 
Example #3
Source Project: metacat   Author: Netflix   File: PartitionUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the partition values from the partition name. This method also validates the partition keys to that
 * of the table.
 *
 * @param tableQName  table name
 * @param table       table
 * @param partName    partition name
 * @return list of partition values
 */
public static List<String> getPartValuesFromPartName(final QualifiedName tableQName, final Table table,
    final String partName) {
    if (Strings.isNullOrEmpty(partName)) {
        throw new InvalidMetaException(tableQName, partName, null);
    }
    final LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
    Warehouse.makeSpecFromName(partSpec, new Path(partName));
    final List<String> values = new ArrayList<>();
    for (FieldSchema field : table.getPartitionKeys()) {
        final String key = field.getName();
        final String val = partSpec.get(key);
        if (val == null) {
            throw new InvalidMetaException(tableQName, partName, null);
        }
        values.add(val);
    }
    return values;
}
 
Example #4
Source Project: incubator-gobblin   Author: apache   File: LocalHiveMetastoreTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception {
  StorageDescriptor partitionSd = new StorageDescriptor();
  if (StringUtils.isNotBlank(tbl.getSd().getLocation())) {
    partitionSd.setLocation(tbl.getSd().getLocation() + values);
  } else {
    partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1");
  }

  partitionSd.setSerdeInfo(
      new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy")));
  partitionSd.setCols(tbl.getPartitionKeys());
  Partition partition =
      new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>());
  partition.setCreateTime(createTime);
  return this.getLocalMetastoreClient().add_partition(partition);

}
 
Example #5
Source Project: waggle-dance   Author: HotelsDotCom   File: DatabaseMappingImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void transformOutboundGetTablesResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  Table table2 = new Table();
  table2.setDbName(DB_NAME);
  table2.setTableName(TABLE_NAME);
  table2.setViewExpandedText(VIEW_EXPANDED_TEXT);
  table2.setViewOriginalText(VIEW_ORIGINAL_TEXT);
  GetTablesResult result = new GetTablesResult();
  result.setTables(Arrays.asList(table, table2));
  GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult.getTables().size(), is(2));
  assertThat(transformedResult.getTables().get(0), is(sameInstance(result.getTables().get(0))));
  assertThat(transformedResult.getTables().get(0).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(0).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(0).getViewExpandedText(), nullValue());
  assertThat(transformedResult.getTables().get(0).getViewOriginalText(), nullValue());
  assertThat(transformedResult.getTables().get(1), is(sameInstance(result.getTables().get(1))));
  assertThat(transformedResult.getTables().get(1).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(1).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(1).getViewExpandedText(), is(VIEW_EXPANDED_TEXT_TRANSFORMED));
  assertThat(transformedResult.getTables().get(1).getViewOriginalText(), is(VIEW_ORIGINAL_TEXT_TRANSFORMED));
}
 
Example #6
Source Project: beeju   Author: HotelsDotCom   File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example #7
Source Project: iceberg   Author: apache   File: HiveTableOperations.java    License: Apache License 2.0 6 votes vote down vote up
private void setParameters(String newMetadataLocation, Table tbl) {
  Map<String, String> parameters = tbl.getParameters();

  if (parameters == null) {
    parameters = new HashMap<>();
  }

  parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ENGLISH));
  parameters.put(METADATA_LOCATION_PROP, newMetadataLocation);

  if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
    parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
  }

  tbl.setParameters(parameters);
}
 
Example #8
Source Project: dremio-oss   Author: dremio   File: HiveClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    default:
      return null;
  }
}
 
Example #9
Source Project: circus-train   Author: HotelsDotCom   File: CopyPartitionsOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 
Example #10
Source Project: incubator-gobblin   Author: apache   File: HiveSchemaEvolutionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEvolutionDisabledForNewTable() throws IOException {
  boolean isEvolutionEnabled = false;
  Optional<Table> destinationTableMeta = Optional.absent();

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, true, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 
Example #11
Source Project: HiveKudu-Handler   Author: BimalTandel   File: KuduStorageHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void rollbackCreateTable(Table tbl) throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if ( client.tableExists(tablename) && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table while rollback of create table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example #12
Source Project: pxf   Author: greenplum-db   File: MultipleHiveFragmentsPerFileFragmenter.java    License: Apache License 2.0 6 votes vote down vote up
private String getFilePath(Table tbl) throws Exception {

        StorageDescriptor descTable = tbl.getSd();

        InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(descTable.getInputFormat(), jobConf);

        FileInputFormat.setInputPaths(jobConf, new Path(descTable.getLocation()));

        InputSplit[] splits;
        try {
            splits = fformat.getSplits(jobConf, 1);
        } catch (org.apache.hadoop.mapred.InvalidInputException e) {
            LOG.debug("getSplits failed on " + e.getMessage());
            throw new RuntimeException("Unable to get file path for table.");
        }

        for (InputSplit split : splits) {
            FileSplit fsp = (FileSplit) split;
            String[] hosts = fsp.getLocations();
            String filepath = fsp.getPath().toString();
            return filepath;
        }
        throw new RuntimeException("Unable to get file path for table.");
    }
 
Example #13
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example #14
Source Project: circus-train   Author: HotelsDotCom   File: AlterTableService.java    License: Apache License 2.0 6 votes vote down vote up
public void alterTable(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws Exception {
  List<FieldSchema> oldColumns = oldTable.getSd().getCols();
  List<FieldSchema> newColumns = newTable.getSd().getCols();
  if (hasAnyChangedColumns(oldColumns, newColumns)) {
    LOG
        .info("Found columns that have changed type, attempting to recreate target table with the new columns."
            + "Old columns: {}, new columns: {}", oldColumns, newColumns);
    Table tempTable = new Table(newTable);
    String tempName = newTable.getTableName() + "_temp";
    tempTable.setTableName(tempName);
    try {
      client.createTable(tempTable);
      copyPartitionsOperation.execute(client, newTable, tempTable);
      renameTableOperation.execute(client, tempTable, newTable);
    } finally {
      dropTableService.dropTable(client, tempTable.getDbName(), tempName);
    }
  } else {
    client.alter_table(newTable.getDbName(), newTable.getTableName(), newTable);
  }
}
 
Example #15
Source Project: iceberg   Author: Netflix   File: HiveTablesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreate() throws TException {
  // Table should be created in hive metastore
  final Table table = metastoreClient.getTable(DB_NAME, TABLE_NAME);

  // check parameters are in expected state
  final Map<String, String> parameters = table.getParameters();
  Assert.assertNotNull(parameters);
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP)));
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getTableType()));

  // Ensure the table is pointing to empty location
  Assert.assertEquals(getTableLocation(TABLE_NAME) , table.getSd().getLocation());

  // Ensure it is stored as unpartitioned table in hive.
  Assert.assertEquals(0 , table.getPartitionKeysSize());

  // Only 1 snapshotFile Should exist and no manifests should exist
  Assert.assertEquals(1, metadataVersionFiles(TABLE_NAME).size());
  Assert.assertEquals(0, manifestFiles(TABLE_NAME).size());

  final com.netflix.iceberg.Table icebergTable = new HiveTables(hiveConf).load(DB_NAME, TABLE_NAME);
  // Iceberg schema should match the loaded table
  Assert.assertEquals(schema.asStruct(), icebergTable.schema().asStruct());
}
 
Example #16
Source Project: presto   Author: prestosql   File: InMemoryThriftMetastore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void dropTable(HiveIdentity identity, String databaseName, String tableName, boolean deleteData)
{
    List<String> locations = listAllDataPaths(identity, this, databaseName, tableName);

    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Table table = relations.remove(schemaTableName);
    if (table == null) {
        throw new TableNotFoundException(schemaTableName);
    }
    views.remove(schemaTableName);
    partitions.keySet().removeIf(partitionName -> partitionName.matches(databaseName, tableName));

    // remove data
    if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
        for (String location : locations) {
            if (location != null) {
                File directory = new File(new Path(location).toUri());
                checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory");
                deleteDirectory(directory);
            }
        }
    }
}
 
Example #17
Source Project: circus-train   Author: HotelsDotCom   File: HiveDifferencesIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void newSourcePartition() throws Exception {
  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  File sourcePartition2Location = createPartitionData("part=2", sourceTableUri,
      Arrays.asList("5\troberto", "6\tpedro"));
  Partition sourcePartition2 = newPartition(DATABASE, SOURCE_TABLE, sourceTable.getSd(), Arrays.asList("2"),
      sourcePartition2Location, null, null, false);
  catalog.client().add_partition(sourcePartition2);

  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);

  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();
  verify(diffListener, never()).onChangedTable(anyList());
  verify(diffListener, times(1))
      .onNewPartition("part=2", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=2"));
  verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList());
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}
 
Example #18
Source Project: circus-train   Author: HotelsDotCom   File: Source.java    License: Apache License 2.0 6 votes vote down vote up
public SourceLocationManager getLocationManager(
    Table table,
    List<Partition> partitions,
    String eventId,
    Map<String, Object> copierOptions)
  throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId,
      table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener);
  boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions,
      CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false);
  if (ignoreMissingFolder) {
    return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf());
  }
  return hdfsSnapshotLocationManager;
}
 
Example #19
Source Project: HiveKudu-Handler   Author: BimalTandel   File: KuduStorageHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void commitDropTable(Table tbl, boolean deleteData)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if (deleteData && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example #20
Source Project: incubator-gobblin   Author: apache   File: ConvertibleHiveDatasetTest.java    License: Apache License 2.0 5 votes vote down vote up
public static ConvertibleHiveDataset createTestConvertibleDataset(Config config)
    throws URISyntaxException {
  Table table = getTestTable("db1", "tb1");
  FileSystem mockFs = Mockito.mock(FileSystem.class);
  when(mockFs.getUri()).thenReturn(new URI("test"));
  ConvertibleHiveDataset cd =
      new ConvertibleHiveDataset(mockFs, Mockito.mock(HiveMetastoreClientPool.class), new org.apache.hadoop.hive.ql.metadata.Table(
          table), new Properties(), config);
  return cd;
}
 
Example #21
Source Project: incubator-gobblin   Author: apache   File: HiveSchemaEvolutionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEvolutionDisabledForExistingTable() throws IOException {
  boolean isEvolutionEnabled = false;
  boolean casePreserved = true;
  Optional<Table> destinationTableMeta = createEvolvedDestinationTable(schemaName, "default", "", true);

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, casePreserved, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_disabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 
Example #22
Source Project: spliceengine   Author: splicemachine   File: SMStorageHandler.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void preCreateTable(Table tbl) throws MetaException {

    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    if (isExternal) {
        Log.info("Creating External table for Splice...");
    }

    String inputTableName = tbl.getParameters().get(MRConstants.SPLICE_TABLE_NAME);
    if (inputTableName == null)
        throw new MetaException("Wrong param, you are missing " +
        		MRConstants.SPLICE_TABLE_NAME + " ? ");

    // We can choose to support user define column mapping.
    // But currently I don't think it is necessary
    // We map all columns from Splice Table to Hive Table.
    String connStr = tbl.getParameters().get(MRConstants.SPLICE_JDBC_STR);
    if (connStr == null)
        throw new MetaException("Wrong param, did you mean " +
        		MRConstants.SPLICE_JDBC_STR + " ? ");
    if (sqlUtil == null)
        sqlUtil = SMSQLUtil.getInstance(connStr);
    if (inputTableName != null) {
        inputTableName = inputTableName.trim();
        checkTableExists(inputTableName);
    }
}
 
Example #23
Source Project: circus-train   Author: HotelsDotCom   File: CircusTrainHdfsHdfsIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void unpartitionedTableReplicateAvroSchema() throws Exception {
  helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));
  LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));

  java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test");
  Files.write(sourceAvroSchemaPath, AVRO_SCHEMA_CONTENT.getBytes());
  String avroSchemaUrl = sourceAvroSchemaPath.toString();

  Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE);
  sourceTable.putToParameters("avro.schema.url", avroSchemaUrl);
  sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable);

  exit.expectSystemExitWithStatus(0);
  File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml");
  CircusTrainRunner runner = CircusTrainRunner
      .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation)
      .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(),
          sourceCatalog.driverClassName())
      .replicaMetaStore(replicaCatalog.getThriftConnectionUri())
      .build();

  exit.checkAssertionAfterwards(new Assertion() {
    @Override
    public void checkAssertion() throws Exception {
      Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE);
      String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/";
      String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
      assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl));
      Path copiedSchema = new Path(transformedAvroUrl);
      FileSystem fs = FileSystem.get(replicaCatalog.conf());
      assertTrue(fs.exists(copiedSchema));
      String content = new String(Files.readAllBytes(java.nio.file.Paths.get(copiedSchema.toUri())));
      assertThat(content, is(AVRO_SCHEMA_CONTENT));
    }
  });

  runner.run(config.getAbsolutePath());
}
 
Example #24
Source Project: Kylin   Author: KylinOLAP   File: HiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
 * 
 * Get basic stats of table
 * @param table
 *          - table
 * @param statType
 *          - type of stats
 * @return value of stats
 */
public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
    Map<String, String> params = table.getParameters();
    long result = 0;

    if (params != null) {
        try {
            result = Long.parseLong(params.get(statType));
        } catch (NumberFormatException e) {
            result = 0;
        }
    }
    return result;
}
 
Example #25
Source Project: flink   Author: apache   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private String getPartitionName(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, Table hiveTable) throws PartitionSpecInvalidException {
	List<String> partitionCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partitionVals = getOrderedFullPartitionValues(partitionSpec, partitionCols, tablePath);
	List<String> partKVs = new ArrayList<>();
	for (int i = 0; i < partitionCols.size(); i++) {
		partKVs.add(partitionCols.get(i) + "=" + partitionVals.get(i));
	}
	return String.join("/", partKVs);
}
 
Example #26
Source Project: circus-train   Author: HotelsDotCom   File: HiveDifferences.java    License: Apache License 2.0 5 votes vote down vote up
private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 
Example #27
Source Project: waggle-dance   Author: HotelsDotCom   File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void alter_table_with_cascade() throws TException {
  Table table = new Table();
  handler.alter_table_with_cascade(DB_P, "table", table, true);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).alter_table_with_cascade(DB_P, "table", table, true);
}
 
Example #28
Source Project: circus-train   Author: HotelsDotCom   File: CircusTrainReplicationModeIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
private void addColumnsToReplica(Table replicaTable) throws Exception {
  List<FieldSchema> columns = replicaTable.getSd().getCols();
  columns.add(new FieldSchema("age", "int", ""));
  columns.add(new FieldSchema("colour", "string", ""));
  replicaTable.getSd().setCols(columns);
  replicaCatalog.client().alter_table(replicaTable.getDbName(), replicaTable.getTableName(), replicaTable);
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: HiveStatsUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive table.
 */
public static ColumnStatistics createTableColumnStats(
		Table hiveTable,
		Map<String, CatalogColumnStatisticsDataBase> colStats) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(true, hiveTable.getDbName(), hiveTable.getTableName());
	return createHiveColumnStatistics(colStats, hiveTable.getSd(), desc);
}
 
Example #30
Source Project: incubator-gobblin   Author: apache   File: HiveConvertExtractor.java    License: Apache License 2.0 5 votes vote down vote up
public HiveConvertExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException {
  super(state);

  if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) {
    log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY));
    return;
  }

  if (!(this.hiveDataset instanceof ConvertibleHiveDataset)) {
    throw new IllegalStateException("HiveConvertExtractor is only compatible with ConvertibleHiveDataset");
  }

  ConvertibleHiveDataset convertibleHiveDataset = (ConvertibleHiveDataset) this.hiveDataset;

  try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) {
    Table table = client.get().getTable(this.dbName, this.tableName);

    SchemaAwareHiveTable schemaAwareHiveTable = new SchemaAwareHiveTable(table, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getTableSchemaUrl(), fs));

    SchemaAwareHivePartition schemaAwareHivePartition = null;

    if (this.hiveWorkUnit.getPartitionName().isPresent() && this.hiveWorkUnit.getPartitionSchemaUrl().isPresent()) {

      Partition partition = client.get().getPartition(this.dbName, this.tableName, this.hiveWorkUnit.getPartitionName().get());
      schemaAwareHivePartition =
          new SchemaAwareHivePartition(table, partition, AvroSchemaManager.getSchemaFromUrl(this.hiveWorkUnit.getPartitionSchemaUrl().get(), fs));
    }

    QueryBasedHiveConversionEntity entity =
        new QueryBasedHiveConversionEntity(convertibleHiveDataset, schemaAwareHiveTable, Optional.fromNullable(schemaAwareHivePartition));
    this.conversionEntities.add(entity);
  }

}