Java Code Examples for org.apache.hadoop.hive.metastore.api.Table#setTableName()

static Table createUnpartitionedTable(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String table,
    File location)
  throws TException {
  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 

static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 

/**
 * <p>
 * NOTE: assumes both `from` and `to` exist
 * </p>
 * Renames tables 'from' table into 'to' table, at the end of the operation 'from' will be gone and 'to' will be
 * renamed.
 */
public void execute(CloseableMetaStoreClient client, Table from, Table to) throws Exception {
  LOG
      .info("Renaming table {}.{} to {}.{}", from.getDbName(), from.getTableName(), to.getDbName(),
          to.getTableName());
  Table fromTable = client.getTable(from.getDbName(), from.getTableName());
  Table toTable = client.getTable(to.getDbName(), to.getTableName());
  String fromTableName = fromTable.getTableName();
  String toTableName = toTable.getTableName();
  String toDelete = toTableName + DELETE_ME;
  try {
    fromTable.setTableName(toTableName);
    toTable.setTableName(toDelete);
    client.alter_table(toTable.getDbName(), toTableName, toTable);
    client.alter_table(fromTable.getDbName(), fromTableName, fromTable);
  } finally {
    dropTableService.dropTable(client, toTable.getDbName(), toDelete);
  }
}
 

@Test
public void alterTableCopyPartitionsFails() throws Exception {
  oldTable.setSd(createStorageDescriptor(Arrays.asList(new FieldSchema("colA", "string", "some comment"))));
  newTable.setSd(createStorageDescriptor(Arrays.asList(new FieldSchema("colA", "int", "some comment"))));
  Table tempTable = new Table(newTable);
  tempTable.setTableName(NEW_TABLE_NAME_TEMP);
  TException toBeThrown = new TException("error");
  doThrow(toBeThrown).when(copyPartitionsOperation).execute(client, newTable, tempTable);

  try {
    service.alterTable(client, oldTable, newTable);
    fail("Should have thrown exception.");
  } catch (Exception e) {
    verify(client).createTable(tempTable);
    verify(dropTableService).dropTable(client, NEW_DB_NAME, NEW_TABLE_NAME_TEMP);
    verify(copyPartitionsOperation).execute(client, newTable, tempTable);
    verifyZeroInteractions(renameTableOperation);
    verifyNoMoreInteractions(client);
    assertThat(e, is(toBeThrown));
  }
}
 

private Table newTable() {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  table.setTableType(TableType.EXTERNAL_TABLE.name());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(tableLocation);
  table.setSd(sd);

  HashMap<String, String> parameters = new HashMap<>();
  parameters.put(StatsSetupConst.ROW_COUNT, "1");
  table.setParameters(parameters);

  table.setPartitionKeys(PARTITIONS);
  return table;
}
 

@Before
public void setUp() {
  SourceTable sourceTable = new SourceTable();
  sourceTable.setDatabaseName(DATABASE);
  sourceTable.setTableName(TABLE);
  tableReplication.setSourceTable(sourceTable);
  ReplicaTable replicaTable = new ReplicaTable();
  replicaTable.setDatabaseName(DATABASE);
  replicaTable.setTableName(REPLICA_TABLE);
  tableReplication.setReplicaTable(replicaTable);
  when(replicaMetaStoreClientSupplier.get()).thenReturn(client);
  replica = new DestructiveReplica(replicaMetaStoreClientSupplier, cleanupLocationManager, tableReplication);

  table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(REPLICA_TABLE);
  table.setPartitionKeys(Lists.newArrayList(new FieldSchema("part1", "string", "")));
  Map<String, String> parameters = new HashMap<>();
  parameters.put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), DATABASE + "." + TABLE);
  parameters.put(REPLICATION_EVENT.parameterName(), EVENT_ID);
  table.setParameters(parameters);
  StorageDescriptor sd1 = new StorageDescriptor();
  sd1.setLocation(tableLocation.toString());
  table.setSd(sd1);
}
 

private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 

private static Table createView(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String view,
    String table,
    List<FieldSchema> partitionCols)
  throws TException {
  Table hiveView = new Table();
  hiveView.setDbName(database);
  hiveView.setTableName(view);
  hiveView.setTableType(TableType.VIRTUAL_VIEW.name());
  hiveView.setViewOriginalText(hql(database, table));
  hiveView.setViewExpandedText(expandHql(database, table, DATA_COLUMNS, partitionCols));
  hiveView.setPartitionKeys(partitionCols);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());
  hiveView.setSd(sd);

  metaStoreClient.createTable(hiveView);

  return hiveView;
}
 

/**
 * Convert a {@link HiveTable} into a {@link Table}.
 */
public static Table getTable(HiveTable hiveTable) {
  State props = hiveTable.getProps();
  Table table = new Table();
  table.setDbName(hiveTable.getDbName());
  table.setTableName(hiveTable.getTableName());
  table.setParameters(getParameters(props));
  if (hiveTable.getCreateTime().isPresent()) {
    table.setCreateTime(Ints.checkedCast(hiveTable.getCreateTime().get()));
  }
  if (hiveTable.getLastAccessTime().isPresent()) {
    table.setLastAccessTime(Ints.checkedCast(hiveTable.getLastAccessTime().get()));
  }
  if (hiveTable.getOwner().isPresent()) {
    table.setOwner(hiveTable.getOwner().get());
  }
  if (hiveTable.getRetention().isPresent()) {
    table.setRetention(Ints.checkedCast(hiveTable.getRetention().get()));
  }
  if (hiveTable.getTableType().isPresent()) {
    table.setTableType(hiveTable.getTableType().get());
  } else {
    table.setTableType(DEFAULT_TABLE_TYPE.toString());
  }
  if (table.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
    table.getParameters().put(EXTERNAL, Boolean.TRUE.toString().toUpperCase());
  }
  table.setPartitionKeys(getFieldSchemas(hiveTable.getPartitionKeys()));
  table.setSd(getStorageDescriptor(hiveTable));
  return table;
}
 

@Test
public void transformOutboundGetTablesResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  GetTablesResult result = new GetTablesResult();
  result.setTables(Collections.singletonList(table));
  GetTablesResult transformedResult = databaseMapping.transformOutboundGetTablesResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult, is(result));
}
 

/**
 * {@inheritDoc}.
 */
@Override
public void rename(final String databaseName,
                   final String oldName,
                   final String newdatabadeName,
                   final String newName) throws TException {
    try (HiveMetastoreClient client = createMetastoreClient()) {
        final Table table = client.get_table(databaseName, oldName);
        client.drop_table(databaseName, oldName, false);
        table.setDbName(newdatabadeName);
        table.setTableName(newName);
        client.create_table(table);
    }
}
 

@Test
public void transformOutboundGetTableResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  GetTableResult result = new GetTableResult();
  result.setTable(table);
  GetTableResult transformedResult = databaseMapping.transformOutboundGetTableResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult.getTable(), is(sameInstance(result.getTable())));
  assertThat(transformedResult.getTable().getDbName(), is(OUT_DB_NAME));
  assertThat(transformedResult.getTable().getTableName(), is(TABLE_NAME));
  assertFalse(transformedResult.getTable().isSetViewExpandedText());
  assertFalse(transformedResult.getTable().isSetViewOriginalText());
}
 

@Override
public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
		throws TableNotExistException, TableAlreadyExistException, CatalogException {
	checkNotNull(tablePath, "tablePath cannot be null");
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(newTableName), "newTableName cannot be null or empty");

	try {
		// alter_table() doesn't throw a clear exception when target table doesn't exist.
		// Thus, check the table existence explicitly
		if (tableExists(tablePath)) {
			ObjectPath newPath = new ObjectPath(tablePath.getDatabaseName(), newTableName);
			// alter_table() doesn't throw a clear exception when new table already exists.
			// Thus, check the table existence explicitly
			if (tableExists(newPath)) {
				throw new TableAlreadyExistException(getName(), newPath);
			} else {
				Table table = getHiveTable(tablePath);
				table.setTableName(newTableName);
				client.alter_table(tablePath.getDatabaseName(), tablePath.getObjectName(), table);
			}
		} else if (!ignoreIfNotExists) {
			throw new TableNotExistException(getName(), tablePath);
		}
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to rename table %s", tablePath.getFullName()), e);
	}
}
 

@Test (expected = UnknownDBException.class)
public void alterTableInvalidDB() throws Exception {
  String newType = "newType";
  Table newHiveTable = CatalogToHiveConverter.convertTable(getTestTable(), hiveTable.getDbName());
  newHiveTable.setTableName(hiveTable.getTableName());
  newHiveTable.setTableType(newType);
  metastoreClient.createTable(hiveTable);
  metastoreClient.alter_table(invalidDatabase, hiveTable.getTableName(), newHiveTable);
}
 

@Test
public void transformOutboundGetTableResult() throws Exception {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  GetTableResult result = new GetTableResult();
  result.setTable(table);
  GetTableResult transformedResult = databaseMapping.transformOutboundGetTableResult(result);
  assertThat(transformedResult, is(sameInstance(result)));
  assertThat(transformedResult, is(result));
}
 

@Test
public void alterTableColumnChange() throws Exception {
  oldTable.setSd(createStorageDescriptor(Arrays.asList(new FieldSchema("colA", "string", "some comment"))));
  newTable.setSd(createStorageDescriptor(Arrays.asList(new FieldSchema("colA", "int", "some comment"))));
  Table tempTable = new Table(newTable);
  tempTable.setTableName(NEW_TABLE_NAME_TEMP);

  service.alterTable(client, oldTable, newTable);

  verify(client).createTable(tempTable);
  verify(dropTableService).dropTable(client, NEW_DB_NAME, NEW_TABLE_NAME_TEMP);
  verify(copyPartitionsOperation).execute(client, newTable, tempTable);
  verify(renameTableOperation).execute(client, tempTable, newTable);
  verifyNoMoreInteractions(client);
}
 

private void createTable(
    String databaseName,
    String tableName,
    File tableLocation,
    String sourceTable,
    String sourceLocation,
    boolean addChecksum)
  throws Exception {
  File partition0 = createPartitionData("part=0", tableLocation, Arrays.asList("1\tadam", "2\tsusan"));
  File partition1 = createPartitionData("part=1", tableLocation, Arrays.asList("3\tchun", "4\tkim"));

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);
  table.setTableType(TableType.EXTERNAL_TABLE.name());
  table.setParameters(new HashMap<String, String>());
  if (sourceTable != null) {
    table.getParameters().put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), sourceTable);
  }
  if (sourceLocation != null) {
    table.getParameters().put(CircusTrainTableParameter.SOURCE_LOCATION.parameterName(), sourceLocation);
  }

  List<FieldSchema> partitionColumns = Arrays.asList(PARTITION_COL);
  table.setPartitionKeys(partitionColumns);

  List<FieldSchema> dataColumns = Arrays.asList(FOO_COL, BAR_COL);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(dataColumns);
  sd.setLocation(tableLocation.toURI().toString());
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());

  table.setSd(sd);

  HiveMetaStoreClient client = catalog.client();
  client.createTable(table);
  LOG
      .info(">>>> Partitions added: {}",
          +client
              .add_partitions(Arrays
                  .asList(
                      newPartition(databaseName, tableName, sd, Arrays.asList("0"), partition0, sourceTable,
                          sourceLocation + "part=0", addChecksum),
                      newPartition(databaseName, tableName, sd, Arrays.asList("1"), partition1, sourceTable,
                          sourceLocation + "part=1", addChecksum))));
}
 

private void createReplicaTable() throws Exception {
  File partitionEurope = new File(replicaTableUri, "local_date=2000-01-01");
  File partitionUk = new File(partitionEurope, "local_hour=0");
  File dataFileUk = new File(partitionUk, PART_00000);
  FileUtils.writeStringToFile(dataFileUk, "1\tadam\tlondon\tuk\n2\tsusan\tglasgow\tuk\n");

  File partitionAsia = new File(replicaTableUri, "local_date=2000-01-02");
  File partitionChina = new File(partitionAsia, "local_hour=0");
  File dataFileChina = new File(partitionChina, PART_00000);
  String data = "1\tchun\tbeijing\tchina\n2\tshanghai\tmilan\titaly\n";
  FileUtils.writeStringToFile(dataFileChina, data);

  HiveMetaStoreClient replicaClient = catalog.client();

  Table replica = new Table();
  replica.setDbName(DATABASE);
  replica.setTableName(REPLICA_TABLE);
  replica.setTableType(TableType.EXTERNAL_TABLE.name());
  Map<String, String> parameters = new HashMap<>();
  parameters.put("comment", "comment replica");
  replica.setParameters(parameters);
  List<FieldSchema> partitionColumns = Arrays.asList(new FieldSchema("local_date", "string", ""),
      new FieldSchema("local_hour", "string", ""));
  replica.setPartitionKeys(partitionColumns);

  List<FieldSchema> dataColumns = Arrays.asList(new FieldSchema("id", "bigint", ""),
      new FieldSchema("name", "string", ""), new FieldSchema("city", "string", ""),
      new FieldSchema("country", "string", ""));

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(dataColumns);
  sd.setLocation(replicaTableUri.toURI().toString());
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());

  replica.setSd(sd);

  replicaClient.createTable(replica);
  LOG.info(">>>> Partitions added: {}",
      +replicaClient.add_partitions(
          Arrays.asList(newPartition(REPLICA_TABLE, sd, Arrays.asList("2000-01-01", "0"), partitionUk),
              newPartition(REPLICA_TABLE, sd, Arrays.asList("2000-01-02", "0"), partitionChina))));
}
 

@Before
public void createDatabase() {
  Database database = new Database();
  database.setName("testdb");
  database.setDescription("testdb");
  database.setLocationUri("hdfs://mycluster/user/hive/warehouse/testdb.db");
  Map<String, String> map = new HashMap<>();
  map.put("key", "value");
  database.setParameters(map);
  database.setOwnerName("root");
  database.setOwnerType(PrincipalType.USER);

  Gson gson = new Gson();
  String databaseJson = gson.toJson(database);

  metaStoreApi.createDatabase(databaseJson);
  Response databaseCountResponse = metaStoreApi.getDatabaseCount();
  assertEquals(databaseCountResponse.getStatus(), Response.Status.OK.getStatusCode());
  assertTrue(((String) databaseCountResponse.getEntity()).contains("\"result\":1"));

  Table table = new Table();
  table.setTableName("testtable");
  table.setDbName("testdb");
  table.setOwner("root");
  table.setCreateTime((int) new java.util.Date().getTime() / 1000);
  table.setLastAccessTime((int) new Date().getTime() / 1000);
  table.setRetention(0);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  FieldSchema fieldSchema = new FieldSchema();
  fieldSchema.setName("a");
  fieldSchema.setType("int");
  fieldSchema.setComment("a");
  fieldSchemas.add(fieldSchema);
  sd.setCols(fieldSchemas);
  sd.setLocation("hdfs://mycluster/user/hive/warehouse/testdb.db/testtable");
  sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  sd.setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  sd.setCompressed(false);
  sd.setNumBuckets(-1);
  SerDeInfo serdeInfo = new SerDeInfo();
  serdeInfo.setName("test");
  serdeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  Map<String, String> parametersMap = new HashMap<>();
  parametersMap.put("serialization.format", "|");
  parametersMap.put("field.delim", "|");
  serdeInfo.setParameters(parametersMap);
  sd.setSerdeInfo(serdeInfo);
  table.setSd(sd);
  List<FieldSchema> partitionKeys = new ArrayList<>();
  table.setPartitionKeys(partitionKeys);
  Map<String, String> parameters = new HashMap<>();
  table.setParameters(parameters);
  String viewOriginalText = "";
  table.setViewOriginalText(viewOriginalText);
  String viewExpandedText = "";
  table.setViewExpandedText(viewExpandedText);
  String tableType = "MANAGED_TABLE";
  table.setTableType(tableType);

  String tableJson = gson.toJson(table);
  metaStoreApi.createTable(tableJson);

  Response tableResponse = metaStoreApi.getTable("testdb", "testtable");
  assertEquals(tableResponse.getStatus(), Response.Status.OK.getStatusCode());
  assertTrue(((String) tableResponse.getEntity()).contains("\"tableName\":\"testtable\""));
  Response tableCountResponse = metaStoreApi.getTableCount();
  assertEquals(tableCountResponse.getStatus(), Response.Status.OK.getStatusCode());
  assertTrue(((String) tableCountResponse.getEntity()).contains("\"result\":1"));
}
 

@Before
public void createDatabase() throws InvalidObjectException, MetaException {
  listTables();

  Database database = new Database();
  database.setName("testdb");
  database.setDescription("testdb");
  database.setLocationUri("hdfs://mycluster/user/hive/warehouse/testdb.db");
  Map map = new HashMap();
  map.put("key", "value");
  database.setParameters(map);
  database.setOwnerName("root");
  database.setOwnerType(PrincipalType.USER);
  submarineMetaStore.createDatabase(database);
  assertEquals(1, submarineMetaStore.getDatabaseCount());

  Table table = new Table();
  table.setTableName("testtable");
  table.setDbName("testdb");
  table.setOwner("root");
  table.setCreateTime((int) new Date().getTime() / 1000);
  table.setLastAccessTime((int) new Date().getTime() / 1000);
  table.setRetention(0);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  FieldSchema fieldSchema = new FieldSchema();
  fieldSchema.setName("a");
  fieldSchema.setType("int");
  fieldSchema.setComment("a");
  fieldSchemas.add(fieldSchema);
  sd.setCols(fieldSchemas);
  sd.setLocation("hdfs://mycluster/user/hive/warehouse/testdb.db/testtable");
  sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  sd.setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  sd.setCompressed(false);
  sd.setNumBuckets(-1);
  SerDeInfo serdeInfo = new SerDeInfo();
  serdeInfo.setName("test");
  serdeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  Map<String, String> parametersMap = new HashMap();
  parametersMap.put("serialization.format", "|");
  parametersMap.put("field.delim", "|");
  serdeInfo.setParameters(parametersMap);
  sd.setSerdeInfo(serdeInfo);
  table.setSd(sd);
  List<FieldSchema> partitionKeys = new ArrayList<>();
  table.setPartitionKeys(partitionKeys);
  Map<String, String> parameters = new HashMap<>();
  table.setParameters(parameters);
  String viewOriginalText = "";
  table.setViewOriginalText(viewOriginalText);
  String viewExpandedText = "";
  table.setViewExpandedText(viewExpandedText);
  String tableType = "MANAGED_TABLE";
  table.setTableType(tableType);
  submarineMetaStore.createTable(table);

  Table tableTest = submarineMetaStore.getTable("testdb", "testtable");
  assertEquals("testtable", tableTest.getTableName());
  int tableCount = submarineMetaStore.getTableCount();
  assertEquals(1, tableCount);
}