Java Code Examples for org.apache.hadoop.hive.metastore.api.FieldSchema

The following examples show how to use org.apache.hadoop.hive.metastore.api.FieldSchema. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: data-highway   Source File: AvroHiveTableStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table newHiveTable(
    String databaseName,
    String tableName,
    String partitionColumnName,
    String location,
    Schema schema,
    int version) {

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);

  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  table.putToParameters("EXTERNAL", "TRUE");
  addRoadAnnotations(table);

  URI schemaUri = uriResolver.resolve(schema, table.getTableName(), version);
  table.putToParameters(AVRO_SCHEMA_URL, schemaUri.toString());
  table.putToParameters(AVRO_SCHEMA_VERSION, Integer.toString(version));
  table.setPartitionKeys(Arrays.asList(new FieldSchema(partitionColumnName, "string", null)));

  table.setSd(AvroStorageDescriptorFactory.create(location));

  return table;
}
 
Example 2
Source Project: circus-train   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static String expandHql(
    String database,
    String table,
    List<FieldSchema> dataColumns,
    List<FieldSchema> partitionColumns) {
  List<String> dataColumnNames = toQualifiedColumnNames(table, dataColumns);
  List<String> partitionColumnNames = partitionColumns != null ? toQualifiedColumnNames(table, partitionColumns)
      : ImmutableList.<String>of();
  List<String> colNames = ImmutableList
      .<String>builder()
      .addAll(dataColumnNames)
      .addAll(partitionColumnNames)
      .build();

  String cols = COMMA_JOINER.join(colNames);
  return String.format("SELECT %s FROM `%s`.`%s`", cols, database, table);
}
 
Example 3
Source Project: kite   Source File: HiveUtils.java    License: Apache License 2.0 6 votes vote down vote up
static Table createEmptyTable(String namespace, String name) {
  Table table = new Table();
  table.setDbName(namespace);
  table.setTableName(name);
  table.setPartitionKeys(new ArrayList<FieldSchema>());
  table.setParameters(new HashMap<String, String>());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setSerdeInfo(new SerDeInfo());
  sd.setNumBuckets(-1);
  sd.setBucketCols(new ArrayList<String>());
  sd.setCols(new ArrayList<FieldSchema>());
  sd.setParameters(new HashMap<String, String>());
  sd.setSortCols(new ArrayList<Order>());
  sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  SkewedInfo skewInfo = new SkewedInfo();
  skewInfo.setSkewedColNames(new ArrayList<String>());
  skewInfo.setSkewedColValues(new ArrayList<List<String>>());
  skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
  sd.setSkewedInfo(skewInfo);
  table.setSd(sd);

  return table;
}
 
Example 4
private void assertTable(HiveMetaStoreClient client, Schema schema, String database, String table,
    List<String> expectedData) throws Exception {
  assertThat(client.getAllTables(database).size(), is(1));
  Table hiveTable = client.getTable(database, table);
  List<FieldSchema> cols = hiveTable.getSd().getCols();
  assertThat(cols.size(), is(schema.getFields().size()));
  assertColumnSchema(schema, cols);
  PartitionIterator partitionIterator = new PartitionIterator(client, hiveTable, (short) 1000);
  List<Partition> partitions = new ArrayList<>();
  while (partitionIterator.hasNext()) {
    Partition partition = partitionIterator.next();
    assertColumnSchema(schema, partition.getSd().getCols());
    partitions.add(partition);
  }
  assertThat(partitions.size(), is(2));
  List<String> data = shell.executeQuery("select * from " + database + "." + table);
  assertThat(data.size(), is(expectedData.size()));
  assertThat(data.containsAll(expectedData), is(true));
}
 
Example 5
@Test
public void dropReplacedPartitionsTest() throws Exception {

  Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName");
  table.setTableType("VIRTUAL_VIEW");
  table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", "")));

  Partition part = new Partition();
  part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01"));

  SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null);
  SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null);

  QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition));
  List<ImmutableMap<String, String>> expected =
      ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01"));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);

  // Make sure that a partition itself is not dropped
  Partition replacedSelf = new Partition();
  replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02"));
  replacedSelf.setValues(ImmutableList.of("2016", "02"));

  conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null)));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
}
 
Example 6
public DatePartitionHiveVersionFinder(FileSystem fs, Config config) {

    this.pattern =
        ConfigUtils.getString(config, PARTITION_VALUE_DATE_TIME_PATTERN_KEY, DEFAULT_PARTITION_VALUE_DATE_TIME_PATTERN);

    if (config.hasPath(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)) {
      this.formatter = DateTimeFormat.forPattern(pattern)
          .withZone(DateTimeZone.forID(config.getString(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)));
    } else {
      this.formatter =
          DateTimeFormat.forPattern(pattern).withZone(DateTimeZone.forID(DEFAULT_PARTITION_VALUE_DATE_TIME_TIMEZONE));
    }

    this.partitionKeyName = ConfigUtils.getString(config, PARTITION_KEY_NAME_KEY, DEFAULT_PARTITION_KEY_NAME);
    this.partitionKeyNamePredicate = new Predicate<FieldSchema>() {

      @Override
      public boolean apply(FieldSchema input) {
        return StringUtils.equalsIgnoreCase(input.getName(), DatePartitionHiveVersionFinder.this.partitionKeyName);
      }
    };
  }
 
Example 7
Source Project: incubator-gobblin   Source File: HiveMetaStoreUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * First tries getting the {@code FieldSchema}s from the {@code HiveRegistrationUnit}'s columns, if set.
 * Else, gets the {@code FieldSchema}s from the deserializer.
 */
private static List<FieldSchema> getFieldSchemas(HiveRegistrationUnit unit) {
  List<Column> columns = unit.getColumns();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  if (columns != null && columns.size() > 0) {
    fieldSchemas = getFieldSchemas(columns);
  } else {
    Deserializer deserializer = getDeserializer(unit);
    if (deserializer != null) {
      try {
        fieldSchemas = MetaStoreUtils.getFieldsFromDeserializer(unit.getTableName(), deserializer);
      } catch (SerDeException | MetaException e) {
        LOG.warn("Encountered exception while getting fields from deserializer.", e);
      }
    }
  }
  return fieldSchemas;
}
 
Example 8
Source Project: presto   Source File: MockThriftMetastoreClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getTable(String dbName, String tableName)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) {
        throw new NoSuchObjectException();
    }
    return new Table(
            TEST_TABLE,
            TEST_DATABASE,
            "",
            0,
            0,
            0,
            DEFAULT_STORAGE_DESCRIPTOR,
            ImmutableList.of(new FieldSchema("key", "string", null)),
            ImmutableMap.of(),
            "",
            "",
            TableType.MANAGED_TABLE.name());
}
 
Example 9
Source Project: dremio-oss   Source File: HiveMetadataUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 
Example 10
Source Project: flink   Source File: HiveTableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a Flink's TableSchema from Hive table's columns and partition keys.
 */
public static TableSchema createTableSchema(List<FieldSchema> cols, List<FieldSchema> partitionKeys) {
	List<FieldSchema> allCols = new ArrayList<>(cols);
	allCols.addAll(partitionKeys);

	String[] colNames = new String[allCols.size()];
	DataType[] colTypes = new DataType[allCols.size()];

	for (int i = 0; i < allCols.size(); i++) {
		FieldSchema fs = allCols.get(i);

		colNames[i] = fs.getName();
		colTypes[i] = HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()));
	}

	return TableSchema.builder()
			.fields(colNames, colTypes)
			.build();
}
 
Example 11
Source Project: flink   Source File: HiveTableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create properties info to initialize a SerDe.
 * @param storageDescriptor
 * @return
 */
public static Properties createPropertiesFromStorageDescriptor(StorageDescriptor storageDescriptor) {
	SerDeInfo serDeInfo = storageDescriptor.getSerdeInfo();
	Map<String, String> parameters = serDeInfo.getParameters();
	Properties properties = new Properties();
	properties.setProperty(
			serdeConstants.SERIALIZATION_FORMAT,
			parameters.get(serdeConstants.SERIALIZATION_FORMAT));
	List<String> colTypes = new ArrayList<>();
	List<String> colNames = new ArrayList<>();
	List<FieldSchema> cols = storageDescriptor.getCols();
	for (FieldSchema col: cols){
		colTypes.add(col.getType());
		colNames.add(col.getName());
	}
	properties.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(colNames, String.valueOf(SerDeUtils.COMMA)));
	// Note: serdeConstants.COLUMN_NAME_DELIMITER is not defined in previous Hive. We use a literal to save on shim
	properties.setProperty("column.name.delimite", String.valueOf(SerDeUtils.COMMA));
	properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, DEFAULT_LIST_COLUMN_TYPES_SEPARATOR));
	properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
	properties.putAll(parameters);
	return properties;
}
 
Example 12
public Table createTestAvroTable(String dbName, String tableName, String tableSdLoc,
    Optional<String> partitionFieldName, boolean ignoreDbCreation) throws Exception {
  if (!ignoreDbCreation) {
    createTestDb(dbName);
  }

  Table tbl = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(dbName, tableName);
  tbl.getSd().setLocation(tableSdLoc);
  tbl.getSd().getSerdeInfo().setSerializationLib(AvroSerDe.class.getName());
  tbl.getSd().getSerdeInfo().setParameters(ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"));

  if (partitionFieldName.isPresent()) {
    tbl.addToPartitionKeys(new FieldSchema(partitionFieldName.get(), "string", "some comment"));
  }

  this.localMetastoreClient.createTable(tbl);

  return tbl;
}
 
Example 13
Source Project: pxf   Source File: HiveClientWrapper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Populates the given metadata object with the given table's fields and partitions,
 * The partition fields are added at the end of the table schema.
 * Throws an exception if the table contains unsupported field types.
 * Supported HCatalog types: TINYINT,
 * SMALLINT, INT, BIGINT, BOOLEAN, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP,
 * DATE, DECIMAL, VARCHAR, CHAR.
 *
 * @param tbl      Hive table
 * @param metadata schema of given table
 */
public void getSchema(Table tbl, Metadata metadata) {

    int hiveColumnsSize = tbl.getSd().getColsSize();
    int hivePartitionsSize = tbl.getPartitionKeysSize();

    LOG.debug("Hive table: {} fields. {} partitions.", hiveColumnsSize, hivePartitionsSize);

    // check hive fields
    try {
        List<FieldSchema> hiveColumns = tbl.getSd().getCols();
        for (FieldSchema hiveCol : hiveColumns) {
            metadata.addField(HiveUtilities.mapHiveType(hiveCol));
        }
        // check partition fields
        List<FieldSchema> hivePartitions = tbl.getPartitionKeys();
        for (FieldSchema hivePart : hivePartitions) {
            metadata.addField(HiveUtilities.mapHiveType(hivePart));
        }
    } catch (UnsupportedTypeException e) {
        String errorMsg = "Failed to retrieve metadata for table " + metadata.getItem() + ". " +
                e.getMessage();
        throw new UnsupportedTypeException(errorMsg);
    }
}
 
Example 14
@Test
public void testCheckTableSchemaMappingMissingColumn() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "tinyint", ""));
  cols.add(new FieldSchema("col3", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column mapping for column: col2");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example 15
Source Project: kite   Source File: TestSchemaConversion.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertSchemaWithComplexRecord() {
  // convertSchema returns a list of FieldSchema objects rather than TypeInfo
  List<FieldSchema> fields = HiveSchemaConverter.convertSchema(COMPLEX_RECORD);

  Assert.assertEquals("Field names should match",
      Lists.newArrayList("groupName", "simpleRecords"),
      Lists.transform(fields, GET_NAMES));
  Assert.assertEquals("Field types should match",
      Lists.newArrayList(
          STRING_TYPE_INFO.toString(),
          TypeInfoFactory.getListTypeInfo(
              TypeInfoFactory.getStructTypeInfo(
                  Lists.newArrayList("id", "name"),
                  Lists.newArrayList(
                      INT_TYPE_INFO,
                      STRING_TYPE_INFO))).toString()),
      Lists.transform(fields, GET_TYPE_STRINGS));
}
 
Example 16
Source Project: griffin   Source File: HiveMetastoreServiceJDBCImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTable() throws SQLException {
    String meta = "CREATE EXTERNAL TABLE `default.session_data`(  `session_date` string COMMENT 'this is session date',   `site_id` int COMMENT '',   `guid` string COMMENT '',   `user_id` string COMMENT '')COMMENT 'session_data for session team' PARTITIONED BY (   `dt` string,   `place` int) ROW FORMAT SERDE   'org.apache.hadoop.hive.serde2.avro.AvroSerDe' STORED AS INPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' LOCATION 'hdfs://localhost/session/common/session_data'TBLPROPERTIES (  'COLUMN_STATS_ACCURATE'='false',   'avro.schema.url'='hdfs://localhost/griffin/session/avro/session-data-1.0.avsc',   'transient_lastDdlTime'='1535651637')";
    when(conn.createStatement()).thenReturn(stmt);
    when(stmt.executeQuery(anyString())).thenReturn(rs);
    when(rs.next()).thenReturn(true).thenReturn(false);
    when(rs.getString(anyInt())).thenReturn(meta);

    Table res = serviceJdbc.getTable("default", "session_data");

    assert (res.getDbName().equals("default"));
    assert (res.getTableName().equals("session_data"));
    assert (res.getSd().getLocation().equals("hdfs://localhost/session/common/session_data"));
    List<FieldSchema> fieldSchemas = res.getSd().getCols();
    for (FieldSchema fieldSchema : fieldSchemas) {
        Assert.assertEquals(fieldSchema.getName(),"session_date");
        Assert.assertEquals(fieldSchema.getType(),"string");
        Assert.assertEquals(fieldSchema.getComment(),"this is session date");
        break;
    }
}
 
Example 17
@Test
public void testDroppedPartitions() throws Exception {
  WorkUnitState previousWus = new WorkUnitState();
  previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "[email protected]_dataset_urn");
  previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true);
  previousWus
      .setActualHighWatermark(new MultiKeyValueLongWatermark(ImmutableMap.of("2015-01", 100l, "2015-02", 101l)));

  SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus));
  PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state);

  Table table = mockTable("test_dataset_urn");
  Mockito.when(table.getPartitionKeys()).thenReturn(ImmutableList.of(new FieldSchema("year", "string", "")));

  Partition partition2015 = mockPartition(table, ImmutableList.of("2015"));

  // partition 2015 replaces 2015-01 and 2015-02
  Mockito.when(partition2015.getParameters()).thenReturn(
      ImmutableMap.of(AbstractAvroToOrcConverter.REPLACED_PARTITIONS_HIVE_METASTORE_KEY, "2015-01|2015-02"));
  watermarker.onPartitionProcessBegin(partition2015, 0l, 0l);

  Assert.assertEquals(watermarker.getExpectedHighWatermarks().get("[email protected]_dataset_urn"), ImmutableMap.of("2015", 0l));
}
 
Example 18
private void setupHiveTables() throws TException, IOException {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("p1"), newFieldSchema("p2"));

  File tableLocation = new File("db1", "table1");
  StorageDescriptor sd = newStorageDescriptor(tableLocation, "col0");
  table1 = newTable("table1", "db1", partitionKeys, sd);
  Partition partition1 = newPartition(table1, "value1", "value2");
  Partition partition2 = newPartition(table1, "value11", "value22");
  table1Partitions = Arrays.asList(partition1, partition2); //
  table1PartitionNames = Arrays
      .asList(Warehouse.makePartName(partitionKeys, partition1.getValues()),
          Warehouse.makePartName(partitionKeys, partition2.getValues()));

  File tableLocation2 = new File("db2", "table2");
  StorageDescriptor sd2 = newStorageDescriptor(tableLocation2, "col0");
  table2 = newTable("table2", "db2", partitionKeys, sd2);
}
 
Example 19
@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 
Example 20
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private List<FieldSchema> getNonPartitionFields(HiveConf hiveConf, Table hiveTable) {
	if (org.apache.hadoop.hive.ql.metadata.Table.hasMetastoreBasedSchema(hiveConf,
			hiveTable.getSd().getSerdeInfo().getSerializationLib())) {
		// get schema from metastore
		return hiveTable.getSd().getCols();
	} else {
		// get schema from deserializer
		return hiveShim.getFieldsFromDeserializer(hiveConf, hiveTable, true);
	}
}
 
Example 21
Source Project: incubator-gobblin   Source File: HiveMetaStoreUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static List<FieldSchema> getFieldSchemas(List<Column> columns) {
  List<FieldSchema> fieldSchemas = Lists.newArrayListWithCapacity(columns.size());
  for (Column column : columns) {
    fieldSchemas.add(new FieldSchema(column.getName(), column.getType(), column.getComment()));
  }
  return fieldSchemas;
}
 
Example 22
Source Project: circus-train   Source File: PartitionsAndStatisticsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void emptyListOfPartitions() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"));
  List<Partition> partitions = Lists.newArrayList();

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);

  assertThat(partitionsAndStatistics.getPartitionNames(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitions(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
}
 
Example 23
Source Project: circus-train   Source File: HiveEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
private List<String> getColumnNames(Table table) {
  List<FieldSchema> fields = table.getSd().getCols();
  List<String> columnNames = new ArrayList<>(fields.size());
  for (FieldSchema field : fields) {
    columnNames.add(field.getName());
  }
  return columnNames;
}
 
Example 24
Source Project: metacat   Source File: HiveConvertersImpl.java    License: Apache License 2.0 5 votes vote down vote up
private FieldDto hiveToMetacatField(final FieldSchema field, final boolean isPartitionKey) {
    final FieldDto dto = new FieldDto();
    dto.setName(field.getName());
    dto.setType(field.getType());
    dto.setSource_type(field.getType());
    dto.setComment(field.getComment());
    dto.setPartition_key(isPartitionKey);

    return dto;
}
 
Example 25
Source Project: circus-train   Source File: FieldSchemaComparatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void typeFullComparison() {
  left.setType("int");
  List<Diff<Object, Object>> diffs = new FieldSchemaComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(1));
  assertThat(diffs.get(0), is(newPropertyDiff(FieldSchema.class, "type", "int", "string")));
}
 
Example 26
@Before
public void setUp() {
  List<FieldSchema> partitionKeys = ImmutableList.of(
        makeFieldSchema("name", "string"),
        makeFieldSchema("birthday", "date"),
        makeFieldSchema("age", "int")
  );

  table = mock(org.apache.hadoop.hive.metastore.api.Table.class);
  when(table.getPartitionKeys()).thenReturn(partitionKeys);
}
 
Example 27
@Before
public void setUp() {
  listener = new PartitionSpecCreatingDiffListener(conf);
  source = new TableAndMetadata(DB, "/tmp", sourceTable);
  replica = Optional.of(new TableAndMetadata(DB, "/tmp", replicaTable));
  when(partitionField1.getName()).thenReturn("p1");
  when(partitionField1.getType()).thenReturn("string");
  when(partitionField2.getName()).thenReturn("p2");
  when(partitionField2.getType()).thenReturn("smallint");
  List<FieldSchema> partitionFields = Lists.newArrayList(partitionField1, partitionField2);
  when(sourceTable.getPartitionKeys()).thenReturn(partitionFields);
}
 
Example 28
Source Project: metacat   Source File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public PartitionsByExprResult get_partitions_by_expr(final PartitionsByExprRequest req) throws TException {
    return requestWrapper("get_partitions_by_expr", new Object[]{req},
        () -> {
            try {
                String filter = null;
                if (req.getExpr() != null) {
                    filter = Utilities.deserializeExpressionFromKryo(req.getExpr()).getExprString();
                    if (filter == null) {
                        throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
                    }
                }
                //TODO: We need to handle the case for 'hasUnknownPartitions'
                return new PartitionsByExprResult(
                    getPartitionsByFilter(req.getDbName(), req.getTblName(), filter, req.getMaxParts()),
                    false);
            } catch (Exception e) {
                //
                // If there is an exception with filtering, fallback to getting all partition names and then
                // apply the filter.
                //
                final List<String> partitionNames = Lists.newArrayList(
                    get_partition_names(req.getDbName(), req.getTblName(), (short) -1));
                final Table table = get_table(req.getDbName(), req.getTblName());
                final List<String> columnNames = new ArrayList<>();
                final List<PrimitiveTypeInfo> typeInfos = new ArrayList<>();
                for (FieldSchema fs : table.getPartitionKeys()) {
                    columnNames.add(fs.getName());
                    typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
                }
                final boolean hasUnknownPartitions = new PartitionExpressionForMetastore().filterPartitionsByExpr(
                    columnNames, typeInfos, req.getExpr(), req.getDefaultPartitionName(), partitionNames);

                return new PartitionsByExprResult(get_partitions_by_names(
                    req.getDbName(), req.getTblName(), partitionNames), hasUnknownPartitions);
            }
        });
}
 
Example 29
Source Project: metacat   Source File: HiveConvertersImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) {
    final TableDto dto = new TableDto();
    dto.setSerde(toStorageDto(table.getSd(), table.getOwner()));
    dto.setAudit(new AuditDto());
    dto.setName(name);
    if (table.isSetCreateTime()) {
        dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime()));
    }
    dto.setMetadata(table.getParameters());

    final List<FieldSchema> nonPartitionColumns = table.getSd().getCols();
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final List<FieldDto> allFields =
        Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, false))
        .forEachOrdered(allFields::add);
    partitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, true))
        .forEachOrdered(allFields::add);
    dto.setFields(allFields);
    dto.setView(new ViewDto(table.getViewOriginalText(),
        table.getViewExpandedText()));
    return dto;
}
 
Example 30
@Test
public void sdColsSameNumberOfElementsFullComparison() {
  left.getPartition().getSd().setCols(
      ImmutableList.of(new FieldSchema("left1", "type", "comment1"), new FieldSchema("left2", "type", "comment2")));
  List<Diff<Object, Object>> diffs = newPartitionAndMetadataComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(4));
  assertThat(diffs.get(0),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getName(),
          right.getPartition().getSd().getCols().get(0).getName())));
  assertThat(diffs.get(1),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getType(),
          right.getPartition().getSd().getCols().get(0).getType())));
  assertThat(diffs.get(2),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getName(),
          right.getPartition().getSd().getCols().get(1).getName())));
  assertThat(diffs.get(3),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getType(),
          right.getPartition().getSd().getCols().get(1).getType())));
}