org.apache.hadoop.hive.metastore.api.FieldSchema Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.FieldSchema. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: data-highway   Author: HotelsDotCom   File: AvroHiveTableStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table newHiveTable(
    String databaseName,
    String tableName,
    String partitionColumnName,
    String location,
    Schema schema,
    int version) {

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);

  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  table.putToParameters("EXTERNAL", "TRUE");
  addRoadAnnotations(table);

  URI schemaUri = uriResolver.resolve(schema, table.getTableName(), version);
  table.putToParameters(AVRO_SCHEMA_URL, schemaUri.toString());
  table.putToParameters(AVRO_SCHEMA_VERSION, Integer.toString(version));
  table.setPartitionKeys(Arrays.asList(new FieldSchema(partitionColumnName, "string", null)));

  table.setSd(AvroStorageDescriptorFactory.create(location));

  return table;
}
 
Example #2
Source Project: circus-train   Author: HotelsDotCom   File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static String expandHql(
    String database,
    String table,
    List<FieldSchema> dataColumns,
    List<FieldSchema> partitionColumns) {
  List<String> dataColumnNames = toQualifiedColumnNames(table, dataColumns);
  List<String> partitionColumnNames = partitionColumns != null ? toQualifiedColumnNames(table, partitionColumns)
      : ImmutableList.<String>of();
  List<String> colNames = ImmutableList
      .<String>builder()
      .addAll(dataColumnNames)
      .addAll(partitionColumnNames)
      .build();

  String cols = COMMA_JOINER.join(colNames);
  return String.format("SELECT %s FROM `%s`.`%s`", cols, database, table);
}
 
Example #3
Source Project: kite   Author: kite-sdk   File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
static Table createEmptyTable(String namespace, String name) {
  Table table = new Table();
  table.setDbName(namespace);
  table.setTableName(name);
  table.setPartitionKeys(new ArrayList<FieldSchema>());
  table.setParameters(new HashMap<String, String>());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setSerdeInfo(new SerDeInfo());
  sd.setNumBuckets(-1);
  sd.setBucketCols(new ArrayList<String>());
  sd.setCols(new ArrayList<FieldSchema>());
  sd.setParameters(new HashMap<String, String>());
  sd.setSortCols(new ArrayList<Order>());
  sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  SkewedInfo skewInfo = new SkewedInfo();
  skewInfo.setSkewedColNames(new ArrayList<String>());
  skewInfo.setSkewedColValues(new ArrayList<List<String>>());
  skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
  sd.setSkewedInfo(skewInfo);
  table.setSd(sd);

  return table;
}
 
Example #4
Source Project: circus-train   Author: HotelsDotCom   File: CircusTrainParquetSchemaEvolutionIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
private void assertTable(HiveMetaStoreClient client, Schema schema, String database, String table,
    List<String> expectedData) throws Exception {
  assertThat(client.getAllTables(database).size(), is(1));
  Table hiveTable = client.getTable(database, table);
  List<FieldSchema> cols = hiveTable.getSd().getCols();
  assertThat(cols.size(), is(schema.getFields().size()));
  assertColumnSchema(schema, cols);
  PartitionIterator partitionIterator = new PartitionIterator(client, hiveTable, (short) 1000);
  List<Partition> partitions = new ArrayList<>();
  while (partitionIterator.hasNext()) {
    Partition partition = partitionIterator.next();
    assertColumnSchema(schema, partition.getSd().getCols());
    partitions.add(partition);
  }
  assertThat(partitions.size(), is(2));
  List<String> data = shell.executeQuery("select * from " + database + "." + table);
  assertThat(data.size(), is(expectedData.size()));
  assertThat(data.containsAll(expectedData), is(true));
}
 
Example #5
Source Project: incubator-gobblin   Author: apache   File: HiveAvroToOrcConverterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void dropReplacedPartitionsTest() throws Exception {

  Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName");
  table.setTableType("VIRTUAL_VIEW");
  table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", "")));

  Partition part = new Partition();
  part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01"));

  SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null);
  SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null);

  QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition));
  List<ImmutableMap<String, String>> expected =
      ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01"));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);

  // Make sure that a partition itself is not dropped
  Partition replacedSelf = new Partition();
  replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02"));
  replacedSelf.setValues(ImmutableList.of("2016", "02"));

  conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null)));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
}
 
Example #6
Source Project: incubator-gobblin   Author: apache   File: DatePartitionHiveVersionFinder.java    License: Apache License 2.0 6 votes vote down vote up
public DatePartitionHiveVersionFinder(FileSystem fs, Config config) {

    this.pattern =
        ConfigUtils.getString(config, PARTITION_VALUE_DATE_TIME_PATTERN_KEY, DEFAULT_PARTITION_VALUE_DATE_TIME_PATTERN);

    if (config.hasPath(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)) {
      this.formatter = DateTimeFormat.forPattern(pattern)
          .withZone(DateTimeZone.forID(config.getString(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)));
    } else {
      this.formatter =
          DateTimeFormat.forPattern(pattern).withZone(DateTimeZone.forID(DEFAULT_PARTITION_VALUE_DATE_TIME_TIMEZONE));
    }

    this.partitionKeyName = ConfigUtils.getString(config, PARTITION_KEY_NAME_KEY, DEFAULT_PARTITION_KEY_NAME);
    this.partitionKeyNamePredicate = new Predicate<FieldSchema>() {

      @Override
      public boolean apply(FieldSchema input) {
        return StringUtils.equalsIgnoreCase(input.getName(), DatePartitionHiveVersionFinder.this.partitionKeyName);
      }
    };
  }
 
Example #7
Source Project: incubator-gobblin   Author: apache   File: HiveMetaStoreUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * First tries getting the {@code FieldSchema}s from the {@code HiveRegistrationUnit}'s columns, if set.
 * Else, gets the {@code FieldSchema}s from the deserializer.
 */
private static List<FieldSchema> getFieldSchemas(HiveRegistrationUnit unit) {
  List<Column> columns = unit.getColumns();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  if (columns != null && columns.size() > 0) {
    fieldSchemas = getFieldSchemas(columns);
  } else {
    Deserializer deserializer = getDeserializer(unit);
    if (deserializer != null) {
      try {
        fieldSchemas = MetaStoreUtils.getFieldsFromDeserializer(unit.getTableName(), deserializer);
      } catch (SerDeException | MetaException e) {
        LOG.warn("Encountered exception while getting fields from deserializer.", e);
      }
    }
  }
  return fieldSchemas;
}
 
Example #8
Source Project: presto   Author: prestosql   File: MockThriftMetastoreClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(String dbName, String tableName)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) {
        throw new NoSuchObjectException();
    }
    return new Table(
            TEST_TABLE,
            TEST_DATABASE,
            "",
            0,
            0,
            0,
            DEFAULT_STORAGE_DESCRIPTOR,
            ImmutableList.of(new FieldSchema("key", "string", null)),
            ImmutableMap.of(),
            "",
            "",
            TableType.MANAGED_TABLE.name());
}
 
Example #9
Source Project: dremio-oss   Author: dremio   File: HiveMetadataUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: HiveTableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a Flink's TableSchema from Hive table's columns and partition keys.
 */
public static TableSchema createTableSchema(List<FieldSchema> cols, List<FieldSchema> partitionKeys) {
	List<FieldSchema> allCols = new ArrayList<>(cols);
	allCols.addAll(partitionKeys);

	String[] colNames = new String[allCols.size()];
	DataType[] colTypes = new DataType[allCols.size()];

	for (int i = 0; i < allCols.size(); i++) {
		FieldSchema fs = allCols.get(i);

		colNames[i] = fs.getName();
		colTypes[i] = HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()));
	}

	return TableSchema.builder()
			.fields(colNames, colTypes)
			.build();
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: HiveTableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create properties info to initialize a SerDe.
 * @param storageDescriptor
 * @return
 */
public static Properties createPropertiesFromStorageDescriptor(StorageDescriptor storageDescriptor) {
	SerDeInfo serDeInfo = storageDescriptor.getSerdeInfo();
	Map<String, String> parameters = serDeInfo.getParameters();
	Properties properties = new Properties();
	properties.setProperty(
			serdeConstants.SERIALIZATION_FORMAT,
			parameters.get(serdeConstants.SERIALIZATION_FORMAT));
	List<String> colTypes = new ArrayList<>();
	List<String> colNames = new ArrayList<>();
	List<FieldSchema> cols = storageDescriptor.getCols();
	for (FieldSchema col: cols){
		colTypes.add(col.getType());
		colNames.add(col.getName());
	}
	properties.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(colNames, String.valueOf(SerDeUtils.COMMA)));
	// Note: serdeConstants.COLUMN_NAME_DELIMITER is not defined in previous Hive. We use a literal to save on shim
	properties.setProperty("column.name.delimite", String.valueOf(SerDeUtils.COMMA));
	properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, DEFAULT_LIST_COLUMN_TYPES_SEPARATOR));
	properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
	properties.putAll(parameters);
	return properties;
}
 
Example #12
Source Project: incubator-gobblin   Author: apache   File: LocalHiveMetastoreTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public Table createTestAvroTable(String dbName, String tableName, String tableSdLoc,
    Optional<String> partitionFieldName, boolean ignoreDbCreation) throws Exception {
  if (!ignoreDbCreation) {
    createTestDb(dbName);
  }

  Table tbl = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(dbName, tableName);
  tbl.getSd().setLocation(tableSdLoc);
  tbl.getSd().getSerdeInfo().setSerializationLib(AvroSerDe.class.getName());
  tbl.getSd().getSerdeInfo().setParameters(ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"));

  if (partitionFieldName.isPresent()) {
    tbl.addToPartitionKeys(new FieldSchema(partitionFieldName.get(), "string", "some comment"));
  }

  this.localMetastoreClient.createTable(tbl);

  return tbl;
}
 
Example #13
Source Project: pxf   Author: greenplum-db   File: HiveClientWrapper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Populates the given metadata object with the given table's fields and partitions,
 * The partition fields are added at the end of the table schema.
 * Throws an exception if the table contains unsupported field types.
 * Supported HCatalog types: TINYINT,
 * SMALLINT, INT, BIGINT, BOOLEAN, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP,
 * DATE, DECIMAL, VARCHAR, CHAR.
 *
 * @param tbl      Hive table
 * @param metadata schema of given table
 */
public void getSchema(Table tbl, Metadata metadata) {

    int hiveColumnsSize = tbl.getSd().getColsSize();
    int hivePartitionsSize = tbl.getPartitionKeysSize();

    LOG.debug("Hive table: {} fields. {} partitions.", hiveColumnsSize, hivePartitionsSize);

    // check hive fields
    try {
        List<FieldSchema> hiveColumns = tbl.getSd().getCols();
        for (FieldSchema hiveCol : hiveColumns) {
            metadata.addField(HiveUtilities.mapHiveType(hiveCol));
        }
        // check partition fields
        List<FieldSchema> hivePartitions = tbl.getPartitionKeys();
        for (FieldSchema hivePart : hivePartitions) {
            metadata.addField(HiveUtilities.mapHiveType(hivePart));
        }
    } catch (UnsupportedTypeException e) {
        String errorMsg = "Failed to retrieve metadata for table " + metadata.getItem() + ". " +
                e.getMessage();
        throw new UnsupportedTypeException(errorMsg);
    }
}
 
Example #14
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckTableSchemaMappingMissingColumn() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "tinyint", ""));
  cols.add(new FieldSchema("col3", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column mapping for column: col2");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example #15
Source Project: kite   Author: kite-sdk   File: TestSchemaConversion.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertSchemaWithComplexRecord() {
  // convertSchema returns a list of FieldSchema objects rather than TypeInfo
  List<FieldSchema> fields = HiveSchemaConverter.convertSchema(COMPLEX_RECORD);

  Assert.assertEquals("Field names should match",
      Lists.newArrayList("groupName", "simpleRecords"),
      Lists.transform(fields, GET_NAMES));
  Assert.assertEquals("Field types should match",
      Lists.newArrayList(
          STRING_TYPE_INFO.toString(),
          TypeInfoFactory.getListTypeInfo(
              TypeInfoFactory.getStructTypeInfo(
                  Lists.newArrayList("id", "name"),
                  Lists.newArrayList(
                      INT_TYPE_INFO,
                      STRING_TYPE_INFO))).toString()),
      Lists.transform(fields, GET_TYPE_STRINGS));
}
 
Example #16
Source Project: griffin   Author: apache   File: HiveMetastoreServiceJDBCImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTable() throws SQLException {
    String meta = "CREATE EXTERNAL TABLE `default.session_data`(  `session_date` string COMMENT 'this is session date',   `site_id` int COMMENT '',   `guid` string COMMENT '',   `user_id` string COMMENT '')COMMENT 'session_data for session team' PARTITIONED BY (   `dt` string,   `place` int) ROW FORMAT SERDE   'org.apache.hadoop.hive.serde2.avro.AvroSerDe' STORED AS INPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' LOCATION 'hdfs://localhost/session/common/session_data'TBLPROPERTIES (  'COLUMN_STATS_ACCURATE'='false',   'avro.schema.url'='hdfs://localhost/griffin/session/avro/session-data-1.0.avsc',   'transient_lastDdlTime'='1535651637')";
    when(conn.createStatement()).thenReturn(stmt);
    when(stmt.executeQuery(anyString())).thenReturn(rs);
    when(rs.next()).thenReturn(true).thenReturn(false);
    when(rs.getString(anyInt())).thenReturn(meta);

    Table res = serviceJdbc.getTable("default", "session_data");

    assert (res.getDbName().equals("default"));
    assert (res.getTableName().equals("session_data"));
    assert (res.getSd().getLocation().equals("hdfs://localhost/session/common/session_data"));
    List<FieldSchema> fieldSchemas = res.getSd().getCols();
    for (FieldSchema fieldSchema : fieldSchemas) {
        Assert.assertEquals(fieldSchema.getName(),"session_date");
        Assert.assertEquals(fieldSchema.getType(),"string");
        Assert.assertEquals(fieldSchema.getComment(),"this is session date");
        break;
    }
}
 
Example #17
Source Project: incubator-gobblin   Author: apache   File: PartitionLevelWatermarkerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDroppedPartitions() throws Exception {
  WorkUnitState previousWus = new WorkUnitState();
  previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "[email protected]_dataset_urn");
  previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true);
  previousWus
      .setActualHighWatermark(new MultiKeyValueLongWatermark(ImmutableMap.of("2015-01", 100l, "2015-02", 101l)));

  SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus));
  PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state);

  Table table = mockTable("test_dataset_urn");
  Mockito.when(table.getPartitionKeys()).thenReturn(ImmutableList.of(new FieldSchema("year", "string", "")));

  Partition partition2015 = mockPartition(table, ImmutableList.of("2015"));

  // partition 2015 replaces 2015-01 and 2015-02
  Mockito.when(partition2015.getParameters()).thenReturn(
      ImmutableMap.of(AbstractAvroToOrcConverter.REPLACED_PARTITIONS_HIVE_METASTORE_KEY, "2015-01|2015-02"));
  watermarker.onPartitionProcessBegin(partition2015, 0l, 0l);

  Assert.assertEquals(watermarker.getExpectedHighWatermarks().get("[email protected]_dataset_urn"), ImmutableMap.of("2015", 0l));
}
 
Example #18
Source Project: circus-train   Author: HotelsDotCom   File: DiffGeneratedPartitionPredicateTest.java    License: Apache License 2.0 6 votes vote down vote up
private void setupHiveTables() throws TException, IOException {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("p1"), newFieldSchema("p2"));

  File tableLocation = new File("db1", "table1");
  StorageDescriptor sd = newStorageDescriptor(tableLocation, "col0");
  table1 = newTable("table1", "db1", partitionKeys, sd);
  Partition partition1 = newPartition(table1, "value1", "value2");
  Partition partition2 = newPartition(table1, "value11", "value22");
  table1Partitions = Arrays.asList(partition1, partition2); //
  table1PartitionNames = Arrays
      .asList(Warehouse.makePartName(partitionKeys, partition1.getValues()),
          Warehouse.makePartName(partitionKeys, partition2.getValues()));

  File tableLocation2 = new File("db2", "table2");
  StorageDescriptor sd2 = newStorageDescriptor(tableLocation2, "col0");
  table2 = newTable("table2", "db2", partitionKeys, sd2);
}
 
Example #19
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example #20
Source Project: flink   Author: apache   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private List<FieldSchema> getNonPartitionFields(HiveConf hiveConf, Table hiveTable) {
	if (org.apache.hadoop.hive.ql.metadata.Table.hasMetastoreBasedSchema(hiveConf,
			hiveTable.getSd().getSerdeInfo().getSerializationLib())) {
		// get schema from metastore
		return hiveTable.getSd().getCols();
	} else {
		// get schema from deserializer
		return hiveShim.getFieldsFromDeserializer(hiveConf, hiveTable, true);
	}
}
 
Example #21
Source Project: incubator-gobblin   Author: apache   File: HiveMetaStoreUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static List<FieldSchema> getFieldSchemas(List<Column> columns) {
  List<FieldSchema> fieldSchemas = Lists.newArrayListWithCapacity(columns.size());
  for (Column column : columns) {
    fieldSchemas.add(new FieldSchema(column.getName(), column.getType(), column.getComment()));
  }
  return fieldSchemas;
}
 
Example #22
Source Project: circus-train   Author: HotelsDotCom   File: PartitionsAndStatisticsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void emptyListOfPartitions() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"));
  List<Partition> partitions = Lists.newArrayList();

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);

  assertThat(partitionsAndStatistics.getPartitionNames(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitions(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
}
 
Example #23
Source Project: circus-train   Author: HotelsDotCom   File: HiveEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
private List<String> getColumnNames(Table table) {
  List<FieldSchema> fields = table.getSd().getCols();
  List<String> columnNames = new ArrayList<>(fields.size());
  for (FieldSchema field : fields) {
    columnNames.add(field.getName());
  }
  return columnNames;
}
 
Example #24
Source Project: metacat   Author: Netflix   File: HiveConvertersImpl.java    License: Apache License 2.0 5 votes vote down vote up
private FieldDto hiveToMetacatField(final FieldSchema field, final boolean isPartitionKey) {
    final FieldDto dto = new FieldDto();
    dto.setName(field.getName());
    dto.setType(field.getType());
    dto.setSource_type(field.getType());
    dto.setComment(field.getComment());
    dto.setPartition_key(isPartitionKey);

    return dto;
}
 
Example #25
Source Project: circus-train   Author: HotelsDotCom   File: FieldSchemaComparatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void typeFullComparison() {
  left.setType("int");
  List<Diff<Object, Object>> diffs = new FieldSchemaComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(1));
  assertThat(diffs.get(0), is(newPropertyDiff(FieldSchema.class, "type", "int", "string")));
}
 
Example #26
@Before
public void setUp() {
  List<FieldSchema> partitionKeys = ImmutableList.of(
        makeFieldSchema("name", "string"),
        makeFieldSchema("birthday", "date"),
        makeFieldSchema("age", "int")
  );

  table = mock(org.apache.hadoop.hive.metastore.api.Table.class);
  when(table.getPartitionKeys()).thenReturn(partitionKeys);
}
 
Example #27
Source Project: circus-train   Author: HotelsDotCom   File: PartitionSpecCreatingDiffListenerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  listener = new PartitionSpecCreatingDiffListener(conf);
  source = new TableAndMetadata(DB, "/tmp", sourceTable);
  replica = Optional.of(new TableAndMetadata(DB, "/tmp", replicaTable));
  when(partitionField1.getName()).thenReturn("p1");
  when(partitionField1.getType()).thenReturn("string");
  when(partitionField2.getName()).thenReturn("p2");
  when(partitionField2.getType()).thenReturn("smallint");
  List<FieldSchema> partitionFields = Lists.newArrayList(partitionField1, partitionField2);
  when(sourceTable.getPartitionKeys()).thenReturn(partitionFields);
}
 
Example #28
Source Project: metacat   Author: Netflix   File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public PartitionsByExprResult get_partitions_by_expr(final PartitionsByExprRequest req) throws TException {
    return requestWrapper("get_partitions_by_expr", new Object[]{req},
        () -> {
            try {
                String filter = null;
                if (req.getExpr() != null) {
                    filter = Utilities.deserializeExpressionFromKryo(req.getExpr()).getExprString();
                    if (filter == null) {
                        throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
                    }
                }
                //TODO: We need to handle the case for 'hasUnknownPartitions'
                return new PartitionsByExprResult(
                    getPartitionsByFilter(req.getDbName(), req.getTblName(), filter, req.getMaxParts()),
                    false);
            } catch (Exception e) {
                //
                // If there is an exception with filtering, fallback to getting all partition names and then
                // apply the filter.
                //
                final List<String> partitionNames = Lists.newArrayList(
                    get_partition_names(req.getDbName(), req.getTblName(), (short) -1));
                final Table table = get_table(req.getDbName(), req.getTblName());
                final List<String> columnNames = new ArrayList<>();
                final List<PrimitiveTypeInfo> typeInfos = new ArrayList<>();
                for (FieldSchema fs : table.getPartitionKeys()) {
                    columnNames.add(fs.getName());
                    typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
                }
                final boolean hasUnknownPartitions = new PartitionExpressionForMetastore().filterPartitionsByExpr(
                    columnNames, typeInfos, req.getExpr(), req.getDefaultPartitionName(), partitionNames);

                return new PartitionsByExprResult(get_partitions_by_names(
                    req.getDbName(), req.getTblName(), partitionNames), hasUnknownPartitions);
            }
        });
}
 
Example #29
Source Project: metacat   Author: Netflix   File: HiveConvertersImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) {
    final TableDto dto = new TableDto();
    dto.setSerde(toStorageDto(table.getSd(), table.getOwner()));
    dto.setAudit(new AuditDto());
    dto.setName(name);
    if (table.isSetCreateTime()) {
        dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime()));
    }
    dto.setMetadata(table.getParameters());

    final List<FieldSchema> nonPartitionColumns = table.getSd().getCols();
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final List<FieldDto> allFields =
        Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, false))
        .forEachOrdered(allFields::add);
    partitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, true))
        .forEachOrdered(allFields::add);
    dto.setFields(allFields);
    dto.setView(new ViewDto(table.getViewOriginalText(),
        table.getViewExpandedText()));
    return dto;
}
 
Example #30
Source Project: circus-train   Author: HotelsDotCom   File: PartitionAndMetadataComparatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void sdColsSameNumberOfElementsFullComparison() {
  left.getPartition().getSd().setCols(
      ImmutableList.of(new FieldSchema("left1", "type", "comment1"), new FieldSchema("left2", "type", "comment2")));
  List<Diff<Object, Object>> diffs = newPartitionAndMetadataComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(4));
  assertThat(diffs.get(0),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getName(),
          right.getPartition().getSd().getCols().get(0).getName())));
  assertThat(diffs.get(1),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getType(),
          right.getPartition().getSd().getCols().get(0).getType())));
  assertThat(diffs.get(2),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getName(),
          right.getPartition().getSd().getCols().get(1).getName())));
  assertThat(diffs.get(3),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getType(),
          right.getPartition().getSd().getCols().get(1).getType())));
}