org.apache.hadoop.hive.metastore.TableType Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.TableType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroHiveTableStrategy.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Override
public Table newHiveTable(
    String databaseName,
    String tableName,
    String partitionColumnName,
    String location,
    Schema schema,
    int version) {

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);

  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  table.putToParameters("EXTERNAL", "TRUE");
  addRoadAnnotations(table);

  URI schemaUri = uriResolver.resolve(schema, table.getTableName(), version);
  table.putToParameters(AVRO_SCHEMA_URL, schemaUri.toString());
  table.putToParameters(AVRO_SCHEMA_VERSION, Integer.toString(version));
  table.setPartitionKeys(Arrays.asList(new FieldSchema(partitionColumnName, "string", null)));

  table.setSd(AvroStorageDescriptorFactory.create(location));

  return table;
}
 
Example #2
Source File: GlueMetastoreClientDelegate.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 
Example #3
Source File: HiveShimV230.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 
Example #4
Source File: HiveMetaStoreBridgeTest.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
@Test
public void testImportThatUpdatesRegisteredTable() throws Exception {
    setupDB(hiveClient, TEST_DB_NAME);

    List<Table> hiveTables = setupTables(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME);

    returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME);

    // return existing table
    when(atlasClient.getEntity(HiveDataTypes.HIVE_TABLE.getName(),
        AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, TEST_DB_NAME, TEST_TABLE_NAME)))
        .thenReturn(getEntityReference(HiveDataTypes.HIVE_TABLE.getName(), "82e06b34-9151-4023-aa9d-b82103a50e77"));
    when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference());
    String processQualifiedName = HiveMetaStoreBridge.getTableProcessQualifiedName(CLUSTER_NAME, hiveTables.get(0));
    when(atlasClient.getEntity(HiveDataTypes.HIVE_PROCESS.getName(),
        AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processQualifiedName)).thenReturn(getEntityReference(HiveDataTypes.HIVE_PROCESS.getName(), "82e06b34-9151-4023-aa9d-b82103a50e77"));

    HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient);
    bridge.importHiveMetadata(true);

    // verify update is called on table
    verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
            (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
                    TableType.EXTERNAL_TABLE.name())));
}
 
Example #5
Source File: GlueMetastoreClientDelegateTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test
public void testValidateTableAndCreateDirectoryVirtualView() throws Exception {
  testTbl.setTableType(TableType.VIRTUAL_VIEW.toString());
  testTbl.getStorageDescriptor().setLocation(null);
  org.apache.hadoop.hive.metastore.api.Table hiveTbl = CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName());

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(testDb));
  when(glueClient.getTable(new GetTableRequest()
    .withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
    .thenThrow(EntityNotFoundException.class);

  assertFalse(metastoreClientDelegate.validateNewTableAndCreateDirectory(hiveTbl));
  assertNull(testTbl.getStorageDescriptor().getLocation());
  verify(wh, never()).mkdirs(any(Path.class), anyBoolean());
}
 
Example #6
Source File: ParquetHiveUtil.java    From streamx with Apache License 2.0 6 votes vote down vote up
private Table constructParquetTable(String database, String tableName, Schema schema, Partitioner partitioner) throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(getHiveParquetSerde());
  try {
    table.setInputFormatClass(getHiveParquetInputFormat());
    table.setOutputFormatClass(getHiveParquetOutputFormat());
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  // convert copycat schema schema to Hive columns
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  return table;
}
 
Example #7
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForUnknownTable() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(sd);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);    
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  doThrow(EntityNotFoundException.class).when(glueClient).getTable(any(GetTableRequest.class));
  
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));     
  
  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 
Example #8
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForUnknownPartition() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(sd);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(externalTable));
  doThrow(EntityNotFoundException.class).when(glueClient).getPartition(any(GetPartitionRequest.class));
  
  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 
Example #9
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForInvalidSD() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(null);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(externalTable));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));

  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 
Example #10
Source File: AvroHiveUtil.java    From streamx with Apache License 2.0 6 votes vote down vote up
private Table constructAvroTable(String database, String tableName, Schema schema, Partitioner partitioner)
    throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(avroSerde);
  try {
    table.setInputFormatClass(avroInputFormat);
    table.setOutputFormatClass(avroOutputFormat);
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString());
  return table;
}
 
Example #11
Source File: HiveMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
private Optional<SystemTable> getPropertiesSystemTable(ConnectorSession session, SchemaTableName tableName, SchemaTableName sourceTableName)
{
    Optional<Table> table = metastore.getTable(new HiveIdentity(session), sourceTableName.getSchemaName(), sourceTableName.getTableName());
    if (table.isEmpty() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
        throw new TableNotFoundException(tableName);
    }
    Map<String, String> sortedTableParameters = ImmutableSortedMap.copyOf(table.get().getParameters());
    List<ColumnMetadata> columns = sortedTableParameters.keySet().stream()
            .map(key -> new ColumnMetadata(key, VarcharType.VARCHAR))
            .collect(toImmutableList());
    List<Type> types = columns.stream()
            .map(ColumnMetadata::getType)
            .collect(toImmutableList());
    Iterable<List<Object>> propertyValues = ImmutableList.of(ImmutableList.copyOf(sortedTableParameters.values()));

    return Optional.of(createSystemTable(new ConnectorTableMetadata(sourceTableName, columns), constraint -> new InMemoryRecordSet(types, propertyValues).cursor()));
}
 
Example #12
Source File: HiveClientImpl.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(final String dbName, final String tableName, boolean ignoreAuthzErrors) throws TException{

  Table table = getTableWithoutTableTypeChecking(dbName, tableName, ignoreAuthzErrors);

  if(table == null){
    return null;
  }

  TableType type = TableType.valueOf(table.getTableType());
  switch (type) {
    case EXTERNAL_TABLE:
    case MANAGED_TABLE:
      return table;

    case VIRTUAL_VIEW:
      throw UserException.unsupportedError().message("Hive views are not supported").build(NOPLogger.NOP_LOGGER);
    case INDEX_TABLE:
    default:
      return null;
  }
}
 
Example #13
Source File: TestUtils.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 
Example #14
Source File: TableMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
public TableMetadata(Table table, Map<String, HiveColumnStatistics> columnStatistics)
{
    owner = table.getOwner();
    tableType = table.getTableType();
    dataColumns = table.getDataColumns();
    partitionColumns = table.getPartitionColumns();
    parameters = table.getParameters();

    StorageFormat tableFormat = table.getStorage().getStorageFormat();
    storageFormat = Arrays.stream(HiveStorageFormat.values())
            .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format)))
            .findFirst();
    bucketProperty = table.getStorage().getBucketProperty();
    serdeParameters = table.getStorage().getSerdeParameters();

    if (tableType.equals(TableType.EXTERNAL_TABLE.name())) {
        externalLocation = Optional.of(table.getStorage().getLocation());
    }
    else {
        externalLocation = Optional.empty();
    }

    viewOriginalText = table.getViewOriginalText();
    viewExpandedText = table.getViewExpandedText();
    this.columnStatistics = ImmutableMap.copyOf(requireNonNull(columnStatistics, "columnStatistics is null"));
}
 
Example #15
Source File: TestUtils.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
static Table createUnpartitionedTable(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String table,
    File location)
  throws TException {
  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 
Example #16
Source File: TestBackgroundHiveSplitLoader.java    From presto with Apache License 2.0 6 votes vote down vote up
private static Table table(
        String location,
        List<Column> partitionColumns,
        Optional<HiveBucketProperty> bucketProperty,
        Map<String, String> tableParameters,
        StorageFormat storageFormat)
{
    Table.Builder tableBuilder = Table.builder();
    tableBuilder.getStorageBuilder()
            .setStorageFormat(storageFormat)
            .setLocation(location)
            .setSkewed(false)
            .setBucketProperty(bucketProperty);

    return tableBuilder
            .setDatabaseName("test_dbname")
            .setOwner("testOwner")
            .setTableName("test_table")
            .setTableType(TableType.MANAGED_TABLE.toString())
            .setDataColumns(ImmutableList.of(new Column("col1", HIVE_STRING, Optional.empty())))
            .setParameters(tableParameters)
            .setPartitionColumns(partitionColumns)
            .build();
}
 
Example #17
Source File: HiveRegistrationPolicyBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * A base implementation for creating a non bucketed, external {@link HiveTable} for a {@link Path}.
 *
 * @param path a {@link Path} used to create the {@link HiveTable}.
 * @param dbName the database name for the created {@link HiveTable}.
 * @param tableName the table name for the created {@link HiveTable}.
 * @return a {@link HiveTable}s for the given {@link Path}.
 * @throws IOException
 */
protected HiveTable getTable(Path path, String dbName, String tableName) throws IOException {

  HiveTable.Builder tableBuilder = new HiveTable.Builder().withDbName(dbName).withTableName(tableName);

  if (!this.emptyInputPathFlag) {
    tableBuilder = tableBuilder.withSerdeManaager(HiveSerDeManager.get(this.props));
  }
  HiveTable table = tableBuilder.build();
  table.setLocation(this.fs.makeQualified(getTableLocation(path)).toString());

  if (!this.emptyInputPathFlag) {
    table.setSerDeProps(path);
  }

  // Setting table-level props.
  table.setProps(getRuntimePropsEnrichedTblProps());

  table.setStorageProps(this.props.getStorageProps());
  table.setSerDeProps(this.props.getSerdeProps());
  table.setNumBuckets(-1);
  table.setBucketColumns(Lists.<String> newArrayList());
  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  return table;
}
 
Example #18
Source File: TestUtils.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private static Table createView(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String view,
    String table,
    List<FieldSchema> partitionCols)
  throws TException {
  Table hiveView = new Table();
  hiveView.setDbName(database);
  hiveView.setTableName(view);
  hiveView.setTableType(TableType.VIRTUAL_VIEW.name());
  hiveView.setViewOriginalText(hql(database, table));
  hiveView.setViewExpandedText(expandHql(database, table, DATA_COLUMNS, partitionCols));
  hiveView.setPartitionKeys(partitionCols);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());
  hiveView.setSd(sd);

  metaStoreClient.createTable(hiveView);

  return hiveView;
}
 
Example #19
Source File: MockThriftMetastoreClient.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(String dbName, String tableName)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) {
        throw new NoSuchObjectException();
    }
    return new Table(
            TEST_TABLE,
            TEST_DATABASE,
            "",
            0,
            0,
            0,
            DEFAULT_STORAGE_DESCRIPTOR,
            ImmutableList.of(new FieldSchema("key", "string", null)),
            ImmutableMap.of(),
            "",
            "",
            TableType.MANAGED_TABLE.name());
}
 
Example #20
Source File: HiveShimV2.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 
Example #21
Source File: ReplicaTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private Table newTable() {
  Table table = new Table();
  table.setDbName(DB_NAME);
  table.setTableName(TABLE_NAME);
  table.setTableType(TableType.EXTERNAL_TABLE.name());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(tableLocation);
  table.setSd(sd);

  HashMap<String, String> parameters = new HashMap<>();
  parameters.put(StatsSetupConst.ROW_COUNT, "1");
  table.setParameters(parameters);

  table.setPartitionKeys(PARTITIONS);
  return table;
}
 
Example #22
Source File: HiveDialectITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateTable() throws Exception {
	String location = warehouse + "/external_location";
	tableEnv.executeSql(String.format(
			"create external table tbl1 (d decimal(10,0),ts timestamp) partitioned by (p string) location '%s' tblproperties('k1'='v1')", location));
	Table hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl1"));
	assertEquals(TableType.EXTERNAL_TABLE.toString(), hiveTable.getTableType());
	assertEquals(1, hiveTable.getPartitionKeysSize());
	assertEquals(location, locationPath(hiveTable.getSd().getLocation()));
	assertEquals("v1", hiveTable.getParameters().get("k1"));
	assertFalse(hiveTable.getParameters().containsKey(SqlCreateHiveTable.TABLE_LOCATION_URI));

	tableEnv.executeSql("create table tbl2 (s struct<ts:timestamp,bin:binary>) stored as orc");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl2"));
	assertEquals(TableType.MANAGED_TABLE.toString(), hiveTable.getTableType());
	assertEquals(OrcSerde.class.getName(), hiveTable.getSd().getSerdeInfo().getSerializationLib());
	assertEquals(OrcInputFormat.class.getName(), hiveTable.getSd().getInputFormat());
	assertEquals(OrcOutputFormat.class.getName(), hiveTable.getSd().getOutputFormat());

	tableEnv.executeSql("create table tbl3 (m map<timestamp,binary>) partitioned by (p1 bigint,p2 tinyint) " +
			"row format serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl3"));
	assertEquals(2, hiveTable.getPartitionKeysSize());
	assertEquals(LazyBinarySerDe.class.getName(), hiveTable.getSd().getSerdeInfo().getSerializationLib());

	tableEnv.executeSql("create table tbl4 (x int,y smallint) row format delimited fields terminated by '|' lines terminated by '\n'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl4"));
	assertEquals("|", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.FIELD_DELIM));
	assertEquals("|", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.SERIALIZATION_FORMAT));
	assertEquals("\n", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.LINE_DELIM));

	tableEnv.executeSql("create table tbl5 (m map<bigint,string>) row format delimited collection items terminated by ';' " +
			"map keys terminated by ':'");
	hiveTable = hiveCatalog.getHiveTable(new ObjectPath("default", "tbl5"));
	assertEquals(";", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.COLLECTION_DELIM));
	assertEquals(":", hiveTable.getSd().getSerdeInfo().getParameters().get(serdeConstants.MAPKEY_DELIM));
}
 
Example #23
Source File: ReplicaTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void convertExistingReplicaTableToView() {
  when(mockReplicaLocationManager.getTableLocation()).thenReturn(null);
  when(mockReplicaLocationManager.getPartitionBaseLocation()).thenReturn(null);
  existingReplicaTable.setTableType(TableType.VIRTUAL_VIEW.name());
  existingReplicaTable.getSd().setLocation(null);
  existingPartition.getSd().setLocation(null);
}
 
Example #24
Source File: CircusTrainTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws TException, IOException {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName("source_" + TABLE);
  table.setTableType(TableType.EXTERNAL_TABLE.name());
  table.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(Arrays.asList(new FieldSchema("col1", "string", null)));
  sd.setSerdeInfo(new SerDeInfo());
  table.setSd(sd);

  hive.client().createTable(table);
}
 
Example #25
Source File: ReplicationFactoryImplTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void viewReplicationModeIsMetadataMirror() {
  tableReplication.setReplicationMode(ReplicationMode.METADATA_MIRROR);
  when(table.getTableType()).thenReturn(TableType.VIRTUAL_VIEW.name());
  Replication replication = factory.newInstance(tableReplication);
  assertThat(replication, is(instanceOf(UnpartitionedTableMetadataMirrorReplication.class)));
}
 
Example #26
Source File: AvroHiveTableStrategyTest.java    From data-highway with Apache License 2.0 5 votes vote down vote up
@Test
public void newHiveTable() throws URISyntaxException {
  when(uriResolver.resolve(schema1, TABLE, 1))
      .thenReturn(new URI("https://s3.amazonaws.com/road-schema-bucket/roads/table/schemas/1/table_v1.avsc"));
  doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant();

  Table result = underTest.newHiveTable(DATABASE, TABLE, PARTITION_COLUMN, LOCATION, schema1, 1);

  assertThat(result.getDbName(), is(DATABASE));
  assertThat(result.getTableName(), is(TABLE));
  assertThat(result.getTableType(), is(TableType.EXTERNAL_TABLE.toString()));
  Map<String, String> parameters = result.getParameters();
  assertThat(parameters.get("EXTERNAL"), is("TRUE"));
  assertThat(parameters.get("data-highway.version"), is(DataHighwayVersion.VERSION));
  assertThat(parameters.get("data-highway.last-revision"), is("2018-05-16T09:17:05Z"));
  assertThat(parameters.get(AvroHiveTableStrategy.AVRO_SCHEMA_URL),
      is("https://s3.amazonaws.com/road-schema-bucket/roads/table/schemas/1/table_v1.avsc"));
  assertThat(parameters.get(AvroHiveTableStrategy.AVRO_SCHEMA_VERSION), is("1"));
  List<FieldSchema> partitionKeys = result.getPartitionKeys();
  assertThat(partitionKeys.size(), is(1));
  assertThat(partitionKeys.get(0), is(new FieldSchema(PARTITION_COLUMN, "string", null)));
  StorageDescriptor storageDescriptor = result.getSd();
  assertThat(storageDescriptor.getInputFormat(), is(AvroStorageDescriptorFactory.AVRO_INPUT_FORMAT));
  assertThat(storageDescriptor.getOutputFormat(), is(AvroStorageDescriptorFactory.AVRO_OUTPUT_FORMAT));
  assertThat(storageDescriptor.getLocation(), is(LOCATION));
  assertThat(storageDescriptor.getCols().size(), is(0));
  SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo();
  assertThat(serdeInfo.getSerializationLib(), is(AvroStorageDescriptorFactory.AVRO_SERDE));
}
 
Example #27
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void newTable() {
  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      REPLICA_DATA_DESTINATION, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name()));
  assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE"));
  assertTrue(MetaStoreUtils.isExternalTable(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 
Example #28
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void newView() {
  sourceTableAndStats.getTable().setTableType(TableType.VIRTUAL_VIEW.name());
  sourceTableAndStats.getTable().getSd().setInputFormat(null);
  sourceTableAndStats.getTable().getSd().setOutputFormat(null);
  sourceTableAndStats.getTable().getSd().setLocation(null);

  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
      null, FULL);
  Table replica = replicaAndStats.getTable();

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(nullValue()));
  assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
  assertThat(replica.getSd().getLocation(), is(nullValue()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
  assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
  assertThat(replica.getTableType(), is(TableType.VIRTUAL_VIEW.name()));
  assertTrue(MetaStoreUtils.isView(replica));

  assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
 
Example #29
Source File: ReplicaTableFactory.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void setReplicaTableType(Table source, Table replica) {
  if (TableType.VIRTUAL_VIEW.name().equals(source.getTableType())) {
    replica.setTableType(TableType.VIRTUAL_VIEW.name());
    return;
  }
  // We set the table to external no matter what. We don't want to delete data accidentally when dropping a mirrored
  // table.
  replica.setTableType(TableType.EXTERNAL_TABLE.name());
  replica.putToParameters(EXTERNAL, "TRUE");
}
 
Example #30
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void newViewPartition() {
  sourceTable.setTableType(TableType.VIRTUAL_VIEW.name());
  sourceTable.getSd().setInputFormat(null);
  sourceTable.getSd().setOutputFormat(null);
  sourceTable.getSd().setLocation(null);

  sourcePartition.getSd().setInputFormat(null);
  sourcePartition.getSd().setOutputFormat(null);
  sourcePartition.getSd().setLocation(null);

  Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME, null,
      FULL);

  assertThat(replica.getDbName(), is(sourceTable.getDbName()));
  assertThat(replica.getTableName(), is(sourceTable.getTableName()));
  assertThat(replica.getSd().getInputFormat(), is(nullValue()));
  assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
  assertThat(replica.getSd().getLocation(), is(nullValue()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
}