org.apache.hadoop.hive.metastore.api.Table Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.Table. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduStorageHandler.java    From HiveKudu-Handler with Apache License 2.0 6 votes vote down vote up
@Override
public void commitDropTable(Table tbl, boolean deleteData)
        throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if (deleteData && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example #2
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_fields_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_fields_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);

            if (table == null || table.getSd() == null || table.getSd().getCols() == null) {
                throw new MetaException("Unable to get fields for " + dbName + "." + tableName);
            }
            return table.getSd().getCols();
        });
}
 
Example #3
Source File: PartitionUtil.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the partition values from the partition name. This method also validates the partition keys to that
 * of the table.
 *
 * @param tableQName  table name
 * @param table       table
 * @param partName    partition name
 * @return list of partition values
 */
public static List<String> getPartValuesFromPartName(final QualifiedName tableQName, final Table table,
    final String partName) {
    if (Strings.isNullOrEmpty(partName)) {
        throw new InvalidMetaException(tableQName, partName, null);
    }
    final LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
    Warehouse.makeSpecFromName(partSpec, new Path(partName));
    final List<String> values = new ArrayList<>();
    for (FieldSchema field : table.getPartitionKeys()) {
        final String key = field.getName();
        final String val = partSpec.get(key);
        if (val == null) {
            throw new InvalidMetaException(tableQName, partName, null);
        }
        values.add(val);
    }
    return values;
}
 
Example #4
Source File: LocalHiveMetastoreTestUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception {
  StorageDescriptor partitionSd = new StorageDescriptor();
  if (StringUtils.isNotBlank(tbl.getSd().getLocation())) {
    partitionSd.setLocation(tbl.getSd().getLocation() + values);
  } else {
    partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1");
  }

  partitionSd.setSerdeInfo(
      new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy")));
  partitionSd.setCols(tbl.getPartitionKeys());
  Partition partition =
      new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>());
  partition.setCreateTime(createTime);
  return this.getLocalMetastoreClient().add_partition(partition);

}
 
Example #5
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void transformOutboundGetTablesResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  Table table2 = new Table();
  table2.setDbName(DB_NAME);
  table2.setTableName(TABLE_NAME);
  table2.setViewExpandedText(VIEW_EXPANDED_TEXT);
  table2.setViewOriginalText(VIEW_ORIGINAL_TEXT);
  GetTablesResult result = new GetTablesResult();
  result.setTables(Arrays.asList(table, table2));
  GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult.getTables().size(), is(2));
  assertThat(transformedResult.getTables().get(0), is(sameInstance(result.getTables().get(0))));
  assertThat(transformedResult.getTables().get(0).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(0).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(0).getViewExpandedText(), nullValue());
  assertThat(transformedResult.getTables().get(0).getViewOriginalText(), nullValue());
  assertThat(transformedResult.getTables().get(1), is(sameInstance(result.getTables().get(1))));
  assertThat(transformedResult.getTables().get(1).getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTables().get(1).getTableName(), is(TABLE_NAME));
  assertThat(transformedResult.getTables().get(1).getViewExpandedText(), is(VIEW_EXPANDED_TEXT_TRANSFORMED));
  assertThat(transformedResult.getTables().get(1).getViewOriginalText(), is(VIEW_ORIGINAL_TEXT_TRANSFORMED));
}
 
Example #6
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example #7
Source File: HiveTableOperations.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private void setParameters(String newMetadataLocation, Table tbl) {
  Map<String, String> parameters = tbl.getParameters();

  if (parameters == null) {
    parameters = new HashMap<>();
  }

  parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE.toUpperCase(Locale.ENGLISH));
  parameters.put(METADATA_LOCATION_PROP, newMetadataLocation);

  if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
    parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
  }

  tbl.setParameters(parameters);
}
 
Example #8
Source File: HiveClientImpl.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    default:
      return null;
  }
}
 
Example #9
Source File: CopyPartitionsOperation.java    From circus-train with Apache License 2.0 6 votes vote down vote up
/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 
Example #10
Source File: HiveSchemaEvolutionTest.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
@Test
public void testEvolutionDisabledForNewTable() throws IOException {
  boolean isEvolutionEnabled = false;
  Optional<Table> destinationTableMeta = Optional.absent();

  String ddl = HiveAvroORCQueryGenerator
      .generateCreateTableDDL(outputSchema, schemaName, "file:/user/hive/warehouse/" + schemaName,
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<List<String>>absent(),
          Optional.<Map<String, HiveAvroORCQueryGenerator.COLUMN_SORT_ORDER>>absent(), Optional.<Integer>absent(),
          Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
          null, isEvolutionEnabled, true, destinationTableMeta,
          new HashMap<String, String>());

  Assert.assertEquals(ddl, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.ddl"),
      "Generated DDL did not match expected for evolution disabled");

  String dml = HiveAvroORCQueryGenerator
      .generateTableMappingDML(inputSchema, outputSchema, schemaName, schemaName + "_orc", Optional.<String>absent(),
          Optional.<String>absent(), Optional.<Map<String, String>>absent(), Optional.<Boolean>absent(),
          Optional.<Boolean>absent(), isEvolutionEnabled, destinationTableMeta, rowLimit);

  Assert.assertEquals(dml, ConversionHiveTestUtils.readQueryFromFile(resourceDir,
      "source_schema_evolution_enabled.dml"),
      "Generated DML did not match expected for evolution disabled");
}
 
Example #11
Source File: MultipleHiveFragmentsPerFileFragmenter.java    From pxf with Apache License 2.0 6 votes vote down vote up
private String getFilePath(Table tbl) throws Exception {

        StorageDescriptor descTable = tbl.getSd();

        InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(descTable.getInputFormat(), jobConf);

        FileInputFormat.setInputPaths(jobConf, new Path(descTable.getLocation()));

        InputSplit[] splits;
        try {
            splits = fformat.getSplits(jobConf, 1);
        } catch (org.apache.hadoop.mapred.InvalidInputException e) {
            LOG.debug("getSplits failed on " + e.getMessage());
            throw new RuntimeException("Unable to get file path for table.");
        }

        for (InputSplit split : splits) {
            FileSplit fsp = (FileSplit) split;
            String[] hosts = fsp.getLocations();
            String filepath = fsp.getPath().toString();
            return filepath;
        }
        throw new RuntimeException("Unable to get file path for table.");
    }
 
Example #12
Source File: DynamoDBStorageHandlerTest.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example #13
Source File: AlterTableService.java    From circus-train with Apache License 2.0 6 votes vote down vote up
public void alterTable(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws Exception {
  List<FieldSchema> oldColumns = oldTable.getSd().getCols();
  List<FieldSchema> newColumns = newTable.getSd().getCols();
  if (hasAnyChangedColumns(oldColumns, newColumns)) {
    LOG
        .info("Found columns that have changed type, attempting to recreate target table with the new columns."
            + "Old columns: {}, new columns: {}", oldColumns, newColumns);
    Table tempTable = new Table(newTable);
    String tempName = newTable.getTableName() + "_temp";
    tempTable.setTableName(tempName);
    try {
      client.createTable(tempTable);
      copyPartitionsOperation.execute(client, newTable, tempTable);
      renameTableOperation.execute(client, tempTable, newTable);
    } finally {
      dropTableService.dropTable(client, tempTable.getDbName(), tempName);
    }
  } else {
    client.alter_table(newTable.getDbName(), newTable.getTableName(), newTable);
  }
}
 
Example #14
Source File: HiveTablesTest.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreate() throws TException {
  // Table should be created in hive metastore
  final Table table = metastoreClient.getTable(DB_NAME, TABLE_NAME);

  // check parameters are in expected state
  final Map<String, String> parameters = table.getParameters();
  Assert.assertNotNull(parameters);
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP)));
  Assert.assertTrue(ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(table.getTableType()));

  // Ensure the table is pointing to empty location
  Assert.assertEquals(getTableLocation(TABLE_NAME) , table.getSd().getLocation());

  // Ensure it is stored as unpartitioned table in hive.
  Assert.assertEquals(0 , table.getPartitionKeysSize());

  // Only 1 snapshotFile Should exist and no manifests should exist
  Assert.assertEquals(1, metadataVersionFiles(TABLE_NAME).size());
  Assert.assertEquals(0, manifestFiles(TABLE_NAME).size());

  final com.netflix.iceberg.Table icebergTable = new HiveTables(hiveConf).load(DB_NAME, TABLE_NAME);
  // Iceberg schema should match the loaded table
  Assert.assertEquals(schema.asStruct(), icebergTable.schema().asStruct());
}
 
Example #15
Source File: InMemoryThriftMetastore.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void dropTable(HiveIdentity identity, String databaseName, String tableName, boolean deleteData)
{
    List<String> locations = listAllDataPaths(identity, this, databaseName, tableName);

    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Table table = relations.remove(schemaTableName);
    if (table == null) {
        throw new TableNotFoundException(schemaTableName);
    }
    views.remove(schemaTableName);
    partitions.keySet().removeIf(partitionName -> partitionName.matches(databaseName, tableName));

    // remove data
    if (deleteData && table.getTableType().equals(MANAGED_TABLE.name())) {
        for (String location : locations) {
            if (location != null) {
                File directory = new File(new Path(location).toUri());
                checkArgument(isParentDir(directory, baseDirectory), "Table directory must be inside of the metastore base directory");
                deleteDirectory(directory);
            }
        }
    }
}
 
Example #16
Source File: HiveDifferencesIntegrationTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void newSourcePartition() throws Exception {
  Table sourceTable = catalog.client().getTable(DATABASE, SOURCE_TABLE);
  File sourcePartition2Location = createPartitionData("part=2", sourceTableUri,
      Arrays.asList("5\troberto", "6\tpedro"));
  Partition sourcePartition2 = newPartition(DATABASE, SOURCE_TABLE, sourceTable.getSd(), Arrays.asList("2"),
      sourcePartition2Location, null, null, false);
  catalog.client().add_partition(sourcePartition2);

  Table replicaTable = catalog.client().getTable(DATABASE, REPLICA_TABLE);

  HiveDifferences
      .builder(diffListener)
      .comparatorRegistry(comparatorRegistry)
      .source(configuration, sourceTable, new PartitionIterator(catalog.client(), sourceTable, PARTITION_BATCH_SIZE))
      .replica(Optional.of(replicaTable),
          Optional.of(new BufferedPartitionFetcher(catalog.client(), replicaTable, PARTITION_BATCH_SIZE)))
      .checksumFunction(checksumFunction)
      .build()
      .run();
  verify(diffListener, never()).onChangedTable(anyList());
  verify(diffListener, times(1))
      .onNewPartition("part=2", catalog.client().getPartition(DATABASE, SOURCE_TABLE, "part=2"));
  verify(diffListener, never()).onChangedPartition(anyString(), any(Partition.class), anyList());
  verify(diffListener, never()).onDataChanged(anyString(), any(Partition.class));
}
 
Example #17
Source File: KuduStorageHandler.java    From HiveKudu-Handler with Apache License 2.0 6 votes vote down vote up
@Override
public void rollbackCreateTable(Table tbl) throws MetaException {
    KuduClient client = getKuduClient(tbl.getParameters().get(HiveKuduConstants.MASTER_ADDRESS_NAME));
    String tablename = getKuduTableName(tbl);
    boolean isExternal = MetaStoreUtils.isExternalTable(tbl);
    try {
        if ( client.tableExists(tablename) && !isExternal) {
            client.deleteTable(tablename);
        }
    } catch (Exception ioe) {
        throw new MetaException("Error dropping table while rollback of create table:" +tablename);
    } finally {
        try {
            client.shutdown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example #18
Source File: Source.java    From circus-train with Apache License 2.0 6 votes vote down vote up
public SourceLocationManager getLocationManager(
    Table table,
    List<Partition> partitions,
    String eventId,
    Map<String, Object> copierOptions)
  throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  HdfsSnapshotLocationManager hdfsSnapshotLocationManager = new HdfsSnapshotLocationManager(getHiveConf(), eventId,
      table, partitions, snapshotsDisabled, sourceTableLocation, sourceCatalogListener);
  boolean ignoreMissingFolder = MapUtils.getBooleanValue(copierOptions,
      CopierOptions.IGNORE_MISSING_PARTITION_FOLDER_ERRORS, false);
  if (ignoreMissingFolder) {
    return new FilterMissingPartitionsLocationManager(hdfsSnapshotLocationManager, getHiveConf());
  }
  return hdfsSnapshotLocationManager;
}
 
Example #19
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example #20
Source File: TableTransformationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Before
public void init() {
  table = new Table();
  table.setDbName("database");
  table.setTableName("table");
  table.setTableType("type");

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  table.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null)));
  storageDescriptor.setInputFormat("input_format");
  storageDescriptor.setOutputFormat("output_format");
  storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation("database/table/");
  table.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  table.setParameters(parameters);
}
 
Example #21
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_schema_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_schema_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);
            List<FieldSchema> partitionKeys = Collections.emptyList();
            List<FieldSchema> columns = Collections.emptyList();

            if (table != null && table.getSd() != null && table.getSd().getCols() != null) {
                columns = table.getSd().getCols();
            }

            if (table != null && table.getPartitionKeys() != null) {
                partitionKeys = table.getPartitionKeys();
            }

            if (partitionKeys.isEmpty() && columns.isEmpty()) {
                throw new MetaException(
                    "Table does not have any partition keys or cols: " + dbName + "." + tableName);
            }

            final List<FieldSchema> result = Lists.newArrayListWithCapacity(partitionKeys.size() + columns.size());
            result.addAll(columns);
            result.addAll(partitionKeys);
            return result;
        });
}
 
Example #22
Source File: HiveMetadataFetcherTest.java    From pxf with Apache License 2.0 5 votes vote down vote up
@Test
public void getTableMetadata() throws Exception {

    fetcher = new HiveMetadataFetcher(context, mockConfigurationFactory, fakeHiveClientWrapper);
    String tableName = "cause";

    // mock hive table returned from hive client
    List<FieldSchema> fields = new ArrayList<>();
    fields.add(new FieldSchema("field1", "string", null));
    fields.add(new FieldSchema("field2", "int", null));
    StorageDescriptor sd = new StorageDescriptor();
    sd.setCols(fields);
    sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
    Table hiveTable = new Table();
    hiveTable.setTableType("MANAGED_TABLE");
    hiveTable.setSd(sd);
    hiveTable.setPartitionKeys(new ArrayList<>());
    when(mockHiveClient.getTable("default", tableName)).thenReturn(hiveTable);

    // Get metadata
    metadataList = fetcher.getMetadata(tableName);
    Metadata metadata = metadataList.get(0);

    assertEquals("default.cause", metadata.getItem().toString());

    List<Metadata.Field> resultFields = metadata.getFields();
    assertNotNull(resultFields);
    assertEquals(2, resultFields.size());
    Metadata.Field field = resultFields.get(0);
    assertEquals("field1", field.getName());
    assertEquals("text", field.getType().getTypeName()); // converted type
    field = resultFields.get(1);
    assertEquals("field2", field.getName());
    assertEquals("int4", field.getType().getTypeName());
}
 
Example #23
Source File: HiveDifferences.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 
Example #24
Source File: ReplicaTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void alteringExistingUnpartitionedReplicaTableSucceeds() throws Exception {
  existingReplicaTable.getParameters().put(REPLICATION_EVENT.parameterName(), "previousEventId");
  replica.updateMetadata(EVENT_ID, tableAndStatistics, DB_NAME, TABLE_NAME, mockReplicaLocationManager);
  verify(alterTableService).alterTable(eq(mockMetaStoreClient), eq(existingReplicaTable), any(Table.class));
  verify(mockMetaStoreClient).updateTableColumnStatistics(columnStatistics);
  verify(mockReplicaLocationManager, never()).addCleanUpLocation(anyString(), any(Path.class));
}
 
Example #25
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void newTable() {
  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      REPLICA_DATA_DESTINATION, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name()));
  assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE"));
  assertTrue(MetaStoreUtils.isExternalTable(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 
Example #26
Source File: CircusTrainHdfsHdfsIntegrationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void unpartitionedTableReplicateAvroSchema() throws Exception {
  helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));
  LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE));

  java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test");
  Files.write(sourceAvroSchemaPath, AVRO_SCHEMA_CONTENT.getBytes());
  String avroSchemaUrl = sourceAvroSchemaPath.toString();

  Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE);
  sourceTable.putToParameters("avro.schema.url", avroSchemaUrl);
  sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable);

  exit.expectSystemExitWithStatus(0);
  File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml");
  CircusTrainRunner runner = CircusTrainRunner
      .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation)
      .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(),
          sourceCatalog.driverClassName())
      .replicaMetaStore(replicaCatalog.getThriftConnectionUri())
      .build();

  exit.checkAssertionAfterwards(new Assertion() {
    @Override
    public void checkAssertion() throws Exception {
      Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE);
      String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/";
      String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
      assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl));
      Path copiedSchema = new Path(transformedAvroUrl);
      FileSystem fs = FileSystem.get(replicaCatalog.conf());
      assertTrue(fs.exists(copiedSchema));
      String content = new String(Files.readAllBytes(java.nio.file.Paths.get(copiedSchema.toUri())));
      assertThat(content, is(AVRO_SCHEMA_CONTENT));
    }
  });

  runner.run(config.getAbsolutePath());
}
 
Example #27
Source File: TableParametersTransformation.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private Map<String, String> mergeTableParameters(Map<String, String> tableParameters, Table table) {
  Map<String, String> parameters;
  if (table.getParameters() != null) {
    parameters = new LinkedHashMap<>(table.getParameters());
  } else {
    parameters = new LinkedHashMap<>();
  }
  parameters.putAll(tableParameters);
  return parameters;
}
 
Example #28
Source File: Source.java    From circus-train with Apache License 2.0 5 votes vote down vote up
public SourceLocationManager getLocationManager(Table table, String eventId) throws IOException {
  if (MetaStoreUtils.isView(table)) {
    return new ViewLocationManager();
  }
  return new HdfsSnapshotLocationManager(getHiveConf(), eventId, table, snapshotsDisabled, sourceTableLocation,
      sourceCatalogListener);
}
 
Example #29
Source File: MetacatHMSHandler.java    From metacat with Apache License 2.0 5 votes vote down vote up
@SuppressFBWarnings
private void initializeAddedPartition(
        final Table tbl, final PartitionSpecProxy.PartitionIterator part,
        final boolean madeDir) throws MetaException {
    // set create time
    final long time = System.currentTimeMillis() / 1000;
    part.setCreateTime((int) time);
    if (part.getParameters() == null || part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null) {
        part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time));
    }

    // Inherit table properties into partition properties.
    final Map<String, String> tblParams = tbl.getParameters();
    final String inheritProps = getHiveConf().getVar(HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim();
    // Default value is empty string in which case no properties will be inherited.
    // * implies all properties needs to be inherited
    Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(",")));
    if (inheritKeys.contains("*")) {
        inheritKeys = tblParams.keySet();
    }

    for (String key : inheritKeys) {
        final String paramVal = tblParams.get(key);
        if (null != paramVal) { // add the property only if it exists in table properties
            part.putToParameters(key, paramVal);
        }
    }
}
 
Example #30
Source File: HiveClient.java    From Kylin with Apache License 2.0 5 votes vote down vote up
/**
 * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
 * 
 * Get basic stats of table
 * @param table
 *          - table
 * @param statType
 *          - type of stats
 * @return value of stats
 */
public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
    Map<String, String> params = table.getParameters();
    long result = 0;

    if (params != null) {
        try {
            result = Long.parseLong(params.get(statType));
        } catch (NumberFormatException e) {
            result = 0;
        }
    }
    return result;
}