org.apache.hadoop.hive.metastore.api.FieldSchema Java Examples

@Override
public Table newHiveTable(
    String databaseName,
    String tableName,
    String partitionColumnName,
    String location,
    Schema schema,
    int version) {

  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);

  table.setTableType(TableType.EXTERNAL_TABLE.toString());
  table.putToParameters("EXTERNAL", "TRUE");
  addRoadAnnotations(table);

  URI schemaUri = uriResolver.resolve(schema, table.getTableName(), version);
  table.putToParameters(AVRO_SCHEMA_URL, schemaUri.toString());
  table.putToParameters(AVRO_SCHEMA_VERSION, Integer.toString(version));
  table.setPartitionKeys(Arrays.asList(new FieldSchema(partitionColumnName, "string", null)));

  table.setSd(AvroStorageDescriptorFactory.create(location));

  return table;
}
 

private static String expandHql(
    String database,
    String table,
    List<FieldSchema> dataColumns,
    List<FieldSchema> partitionColumns) {
  List<String> dataColumnNames = toQualifiedColumnNames(table, dataColumns);
  List<String> partitionColumnNames = partitionColumns != null ? toQualifiedColumnNames(table, partitionColumns)
      : ImmutableList.<String>of();
  List<String> colNames = ImmutableList
      .<String>builder()
      .addAll(dataColumnNames)
      .addAll(partitionColumnNames)
      .build();

  String cols = COMMA_JOINER.join(colNames);
  return String.format("SELECT %s FROM `%s`.`%s`", cols, database, table);
}
 

static Table createEmptyTable(String namespace, String name) {
  Table table = new Table();
  table.setDbName(namespace);
  table.setTableName(name);
  table.setPartitionKeys(new ArrayList<FieldSchema>());
  table.setParameters(new HashMap<String, String>());

  StorageDescriptor sd = new StorageDescriptor();
  sd.setSerdeInfo(new SerDeInfo());
  sd.setNumBuckets(-1);
  sd.setBucketCols(new ArrayList<String>());
  sd.setCols(new ArrayList<FieldSchema>());
  sd.setParameters(new HashMap<String, String>());
  sd.setSortCols(new ArrayList<Order>());
  sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  SkewedInfo skewInfo = new SkewedInfo();
  skewInfo.setSkewedColNames(new ArrayList<String>());
  skewInfo.setSkewedColValues(new ArrayList<List<String>>());
  skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
  sd.setSkewedInfo(skewInfo);
  table.setSd(sd);

  return table;
}
 

private void assertTable(HiveMetaStoreClient client, Schema schema, String database, String table,
    List<String> expectedData) throws Exception {
  assertThat(client.getAllTables(database).size(), is(1));
  Table hiveTable = client.getTable(database, table);
  List<FieldSchema> cols = hiveTable.getSd().getCols();
  assertThat(cols.size(), is(schema.getFields().size()));
  assertColumnSchema(schema, cols);
  PartitionIterator partitionIterator = new PartitionIterator(client, hiveTable, (short) 1000);
  List<Partition> partitions = new ArrayList<>();
  while (partitionIterator.hasNext()) {
    Partition partition = partitionIterator.next();
    assertColumnSchema(schema, partition.getSd().getCols());
    partitions.add(partition);
  }
  assertThat(partitions.size(), is(2));
  List<String> data = shell.executeQuery("select * from " + database + "." + table);
  assertThat(data.size(), is(expectedData.size()));
  assertThat(data.containsAll(expectedData), is(true));
}
 

@Test
public void dropReplacedPartitionsTest() throws Exception {

  Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName");
  table.setTableType("VIRTUAL_VIEW");
  table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", "")));

  Partition part = new Partition();
  part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01"));

  SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null);
  SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null);

  QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition));
  List<ImmutableMap<String, String>> expected =
      ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01"));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);

  // Make sure that a partition itself is not dropped
  Partition replacedSelf = new Partition();
  replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02"));
  replacedSelf.setValues(ImmutableList.of("2016", "02"));

  conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null)));
  Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
}
 

public DatePartitionHiveVersionFinder(FileSystem fs, Config config) {

    this.pattern =
        ConfigUtils.getString(config, PARTITION_VALUE_DATE_TIME_PATTERN_KEY, DEFAULT_PARTITION_VALUE_DATE_TIME_PATTERN);

    if (config.hasPath(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)) {
      this.formatter = DateTimeFormat.forPattern(pattern)
          .withZone(DateTimeZone.forID(config.getString(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)));
    } else {
      this.formatter =
          DateTimeFormat.forPattern(pattern).withZone(DateTimeZone.forID(DEFAULT_PARTITION_VALUE_DATE_TIME_TIMEZONE));
    }

    this.partitionKeyName = ConfigUtils.getString(config, PARTITION_KEY_NAME_KEY, DEFAULT_PARTITION_KEY_NAME);
    this.partitionKeyNamePredicate = new Predicate<FieldSchema>() {

      @Override
      public boolean apply(FieldSchema input) {
        return StringUtils.equalsIgnoreCase(input.getName(), DatePartitionHiveVersionFinder.this.partitionKeyName);
      }
    };
  }
 

/**
 * First tries getting the {@code FieldSchema}s from the {@code HiveRegistrationUnit}'s columns, if set.
 * Else, gets the {@code FieldSchema}s from the deserializer.
 */
private static List<FieldSchema> getFieldSchemas(HiveRegistrationUnit unit) {
  List<Column> columns = unit.getColumns();
  List<FieldSchema> fieldSchemas = new ArrayList<>();
  if (columns != null && columns.size() > 0) {
    fieldSchemas = getFieldSchemas(columns);
  } else {
    Deserializer deserializer = getDeserializer(unit);
    if (deserializer != null) {
      try {
        fieldSchemas = MetaStoreUtils.getFieldsFromDeserializer(unit.getTableName(), deserializer);
      } catch (SerDeException | MetaException e) {
        LOG.warn("Encountered exception while getting fields from deserializer.", e);
      }
    }
  }
  return fieldSchemas;
}
 

@Override
public Table getTable(String dbName, String tableName)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!dbName.equals(TEST_DATABASE) || !tableName.equals(TEST_TABLE)) {
        throw new NoSuchObjectException();
    }
    return new Table(
            TEST_TABLE,
            TEST_DATABASE,
            "",
            0,
            0,
            0,
            DEFAULT_STORAGE_DESCRIPTOR,
            ImmutableList.of(new FieldSchema("key", "string", null)),
            ImmutableMap.of(),
            "",
            "",
            TableType.MANAGED_TABLE.name());
}
 

public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 

/**
 * Create a Flink's TableSchema from Hive table's columns and partition keys.
 */
public static TableSchema createTableSchema(List<FieldSchema> cols, List<FieldSchema> partitionKeys) {
	List<FieldSchema> allCols = new ArrayList<>(cols);
	allCols.addAll(partitionKeys);

	String[] colNames = new String[allCols.size()];
	DataType[] colTypes = new DataType[allCols.size()];

	for (int i = 0; i < allCols.size(); i++) {
		FieldSchema fs = allCols.get(i);

		colNames[i] = fs.getName();
		colTypes[i] = HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()));
	}

	return TableSchema.builder()
			.fields(colNames, colTypes)
			.build();
}
 

/**
 * Create properties info to initialize a SerDe.
 * @param storageDescriptor
 * @return
 */
public static Properties createPropertiesFromStorageDescriptor(StorageDescriptor storageDescriptor) {
	SerDeInfo serDeInfo = storageDescriptor.getSerdeInfo();
	Map<String, String> parameters = serDeInfo.getParameters();
	Properties properties = new Properties();
	properties.setProperty(
			serdeConstants.SERIALIZATION_FORMAT,
			parameters.get(serdeConstants.SERIALIZATION_FORMAT));
	List<String> colTypes = new ArrayList<>();
	List<String> colNames = new ArrayList<>();
	List<FieldSchema> cols = storageDescriptor.getCols();
	for (FieldSchema col: cols){
		colTypes.add(col.getType());
		colNames.add(col.getName());
	}
	properties.setProperty(serdeConstants.LIST_COLUMNS, StringUtils.join(colNames, String.valueOf(SerDeUtils.COMMA)));
	// Note: serdeConstants.COLUMN_NAME_DELIMITER is not defined in previous Hive. We use a literal to save on shim
	properties.setProperty("column.name.delimite", String.valueOf(SerDeUtils.COMMA));
	properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, StringUtils.join(colTypes, DEFAULT_LIST_COLUMN_TYPES_SEPARATOR));
	properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
	properties.putAll(parameters);
	return properties;
}
 

public Table createTestAvroTable(String dbName, String tableName, String tableSdLoc,
    Optional<String> partitionFieldName, boolean ignoreDbCreation) throws Exception {
  if (!ignoreDbCreation) {
    createTestDb(dbName);
  }

  Table tbl = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(dbName, tableName);
  tbl.getSd().setLocation(tableSdLoc);
  tbl.getSd().getSerdeInfo().setSerializationLib(AvroSerDe.class.getName());
  tbl.getSd().getSerdeInfo().setParameters(ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"));

  if (partitionFieldName.isPresent()) {
    tbl.addToPartitionKeys(new FieldSchema(partitionFieldName.get(), "string", "some comment"));
  }

  this.localMetastoreClient.createTable(tbl);

  return tbl;
}
 

/**
 * Populates the given metadata object with the given table's fields and partitions,
 * The partition fields are added at the end of the table schema.
 * Throws an exception if the table contains unsupported field types.
 * Supported HCatalog types: TINYINT,
 * SMALLINT, INT, BIGINT, BOOLEAN, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP,
 * DATE, DECIMAL, VARCHAR, CHAR.
 *
 * @param tbl      Hive table
 * @param metadata schema of given table
 */
public void getSchema(Table tbl, Metadata metadata) {

    int hiveColumnsSize = tbl.getSd().getColsSize();
    int hivePartitionsSize = tbl.getPartitionKeysSize();

    LOG.debug("Hive table: {} fields. {} partitions.", hiveColumnsSize, hivePartitionsSize);

    // check hive fields
    try {
        List<FieldSchema> hiveColumns = tbl.getSd().getCols();
        for (FieldSchema hiveCol : hiveColumns) {
            metadata.addField(HiveUtilities.mapHiveType(hiveCol));
        }
        // check partition fields
        List<FieldSchema> hivePartitions = tbl.getPartitionKeys();
        for (FieldSchema hivePart : hivePartitions) {
            metadata.addField(HiveUtilities.mapHiveType(hivePart));
        }
    } catch (UnsupportedTypeException e) {
        String errorMsg = "Failed to retrieve metadata for table " + metadata.getItem() + ". " +
                e.getMessage();
        throw new UnsupportedTypeException(errorMsg);
    }
}
 

@Test
public void testCheckTableSchemaMappingMissingColumn() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "tinyint", ""));
  cols.add(new FieldSchema("col3", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column mapping for column: col2");
  storageHandler.checkTableSchemaMapping(description, table);
}
 

@Test
public void testConvertSchemaWithComplexRecord() {
  // convertSchema returns a list of FieldSchema objects rather than TypeInfo
  List<FieldSchema> fields = HiveSchemaConverter.convertSchema(COMPLEX_RECORD);

  Assert.assertEquals("Field names should match",
      Lists.newArrayList("groupName", "simpleRecords"),
      Lists.transform(fields, GET_NAMES));
  Assert.assertEquals("Field types should match",
      Lists.newArrayList(
          STRING_TYPE_INFO.toString(),
          TypeInfoFactory.getListTypeInfo(
              TypeInfoFactory.getStructTypeInfo(
                  Lists.newArrayList("id", "name"),
                  Lists.newArrayList(
                      INT_TYPE_INFO,
                      STRING_TYPE_INFO))).toString()),
      Lists.transform(fields, GET_TYPE_STRINGS));
}
 

@Test
public void testGetTable() throws SQLException {
    String meta = "CREATE EXTERNAL TABLE `default.session_data`(  `session_date` string COMMENT 'this is session date',   `site_id` int COMMENT '',   `guid` string COMMENT '',   `user_id` string COMMENT '')COMMENT 'session_data for session team' PARTITIONED BY (   `dt` string,   `place` int) ROW FORMAT SERDE   'org.apache.hadoop.hive.serde2.avro.AvroSerDe' STORED AS INPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT   'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' LOCATION 'hdfs://localhost/session/common/session_data'TBLPROPERTIES (  'COLUMN_STATS_ACCURATE'='false',   'avro.schema.url'='hdfs://localhost/griffin/session/avro/session-data-1.0.avsc',   'transient_lastDdlTime'='1535651637')";
    when(conn.createStatement()).thenReturn(stmt);
    when(stmt.executeQuery(anyString())).thenReturn(rs);
    when(rs.next()).thenReturn(true).thenReturn(false);
    when(rs.getString(anyInt())).thenReturn(meta);

    Table res = serviceJdbc.getTable("default", "session_data");

    assert (res.getDbName().equals("default"));
    assert (res.getTableName().equals("session_data"));
    assert (res.getSd().getLocation().equals("hdfs://localhost/session/common/session_data"));
    List<FieldSchema> fieldSchemas = res.getSd().getCols();
    for (FieldSchema fieldSchema : fieldSchemas) {
        Assert.assertEquals(fieldSchema.getName(),"session_date");
        Assert.assertEquals(fieldSchema.getType(),"string");
        Assert.assertEquals(fieldSchema.getComment(),"this is session date");
        break;
    }
}
 

@Test
public void testDroppedPartitions() throws Exception {
  WorkUnitState previousWus = new WorkUnitState();
  previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "[email protected]_dataset_urn");
  previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true);
  previousWus
      .setActualHighWatermark(new MultiKeyValueLongWatermark(ImmutableMap.of("2015-01", 100l, "2015-02", 101l)));

  SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus));
  PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state);

  Table table = mockTable("test_dataset_urn");
  Mockito.when(table.getPartitionKeys()).thenReturn(ImmutableList.of(new FieldSchema("year", "string", "")));

  Partition partition2015 = mockPartition(table, ImmutableList.of("2015"));

  // partition 2015 replaces 2015-01 and 2015-02
  Mockito.when(partition2015.getParameters()).thenReturn(
      ImmutableMap.of(AbstractAvroToOrcConverter.REPLACED_PARTITIONS_HIVE_METASTORE_KEY, "2015-01|2015-02"));
  watermarker.onPartitionProcessBegin(partition2015, 0l, 0l);

  Assert.assertEquals(watermarker.getExpectedHighWatermarks().get("[email protected]_dataset_urn"), ImmutableMap.of("2015", 0l));
}
 

private void setupHiveTables() throws TException, IOException {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("p1"), newFieldSchema("p2"));

  File tableLocation = new File("db1", "table1");
  StorageDescriptor sd = newStorageDescriptor(tableLocation, "col0");
  table1 = newTable("table1", "db1", partitionKeys, sd);
  Partition partition1 = newPartition(table1, "value1", "value2");
  Partition partition2 = newPartition(table1, "value11", "value22");
  table1Partitions = Arrays.asList(partition1, partition2); //
  table1PartitionNames = Arrays
      .asList(Warehouse.makePartName(partitionKeys, partition1.getValues()),
          Warehouse.makePartName(partitionKeys, partition2.getValues()));

  File tableLocation2 = new File("db2", "table2");
  StorageDescriptor sd2 = newStorageDescriptor(tableLocation2, "col0");
  table2 = newTable("table2", "db2", partitionKeys, sd2);
}
 

@Test
public void testCheckTableSchemaMappingMissingColumnMapping() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
   "col2:dynamo_col2#,hashKey:hashKey,hashMap:hashMap");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("hashMap", "map<string,string>", ""));
  sd.setCols(cols);
  table.setSd(sd);

  exceptionRule.expect(MetaException.class);
  exceptionRule.expectMessage("Could not find column(s) for column mapping(s): ");
  exceptionRule.expectMessage("col2:dynamo_col2#");
  exceptionRule.expectMessage("hashkey:hashKey");
  storageHandler.checkTableSchemaMapping(description, table);
}
 

private List<FieldSchema> getNonPartitionFields(HiveConf hiveConf, Table hiveTable) {
	if (org.apache.hadoop.hive.ql.metadata.Table.hasMetastoreBasedSchema(hiveConf,
			hiveTable.getSd().getSerdeInfo().getSerializationLib())) {
		// get schema from metastore
		return hiveTable.getSd().getCols();
	} else {
		// get schema from deserializer
		return hiveShim.getFieldsFromDeserializer(hiveConf, hiveTable, true);
	}
}
 

private static List<FieldSchema> getFieldSchemas(List<Column> columns) {
  List<FieldSchema> fieldSchemas = Lists.newArrayListWithCapacity(columns.size());
  for (Column column : columns) {
    fieldSchemas.add(new FieldSchema(column.getName(), column.getType(), column.getComment()));
  }
  return fieldSchemas;
}
 

@Test
public void emptyListOfPartitions() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"));
  List<Partition> partitions = Lists.newArrayList();

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);

  assertThat(partitionsAndStatistics.getPartitionNames(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitions(), is(empty()));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
}
 

private List<String> getColumnNames(Table table) {
  List<FieldSchema> fields = table.getSd().getCols();
  List<String> columnNames = new ArrayList<>(fields.size());
  for (FieldSchema field : fields) {
    columnNames.add(field.getName());
  }
  return columnNames;
}
 

private FieldDto hiveToMetacatField(final FieldSchema field, final boolean isPartitionKey) {
    final FieldDto dto = new FieldDto();
    dto.setName(field.getName());
    dto.setType(field.getType());
    dto.setSource_type(field.getType());
    dto.setComment(field.getComment());
    dto.setPartition_key(isPartitionKey);

    return dto;
}
 

@Test
public void typeFullComparison() {
  left.setType("int");
  List<Diff<Object, Object>> diffs = new FieldSchemaComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(1));
  assertThat(diffs.get(0), is(newPropertyDiff(FieldSchema.class, "type", "int", "string")));
}
 

@Before
public void setUp() {
  List<FieldSchema> partitionKeys = ImmutableList.of(
        makeFieldSchema("name", "string"),
        makeFieldSchema("birthday", "date"),
        makeFieldSchema("age", "int")
  );

  table = mock(org.apache.hadoop.hive.metastore.api.Table.class);
  when(table.getPartitionKeys()).thenReturn(partitionKeys);
}
 

@Before
public void setUp() {
  listener = new PartitionSpecCreatingDiffListener(conf);
  source = new TableAndMetadata(DB, "/tmp", sourceTable);
  replica = Optional.of(new TableAndMetadata(DB, "/tmp", replicaTable));
  when(partitionField1.getName()).thenReturn("p1");
  when(partitionField1.getType()).thenReturn("string");
  when(partitionField2.getName()).thenReturn("p2");
  when(partitionField2.getType()).thenReturn("smallint");
  List<FieldSchema> partitionFields = Lists.newArrayList(partitionField1, partitionField2);
  when(sourceTable.getPartitionKeys()).thenReturn(partitionFields);
}
 

/**
 * {@inheritDoc}
 */
@Override
public PartitionsByExprResult get_partitions_by_expr(final PartitionsByExprRequest req) throws TException {
    return requestWrapper("get_partitions_by_expr", new Object[]{req},
        () -> {
            try {
                String filter = null;
                if (req.getExpr() != null) {
                    filter = Utilities.deserializeExpressionFromKryo(req.getExpr()).getExprString();
                    if (filter == null) {
                        throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
                    }
                }
                //TODO: We need to handle the case for 'hasUnknownPartitions'
                return new PartitionsByExprResult(
                    getPartitionsByFilter(req.getDbName(), req.getTblName(), filter, req.getMaxParts()),
                    false);
            } catch (Exception e) {
                //
                // If there is an exception with filtering, fallback to getting all partition names and then
                // apply the filter.
                //
                final List<String> partitionNames = Lists.newArrayList(
                    get_partition_names(req.getDbName(), req.getTblName(), (short) -1));
                final Table table = get_table(req.getDbName(), req.getTblName());
                final List<String> columnNames = new ArrayList<>();
                final List<PrimitiveTypeInfo> typeInfos = new ArrayList<>();
                for (FieldSchema fs : table.getPartitionKeys()) {
                    columnNames.add(fs.getName());
                    typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
                }
                final boolean hasUnknownPartitions = new PartitionExpressionForMetastore().filterPartitionsByExpr(
                    columnNames, typeInfos, req.getExpr(), req.getDefaultPartitionName(), partitionNames);

                return new PartitionsByExprResult(get_partitions_by_names(
                    req.getDbName(), req.getTblName(), partitionNames), hasUnknownPartitions);
            }
        });
}
 

/**
 * {@inheritDoc}
 */
@Override
public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) {
    final TableDto dto = new TableDto();
    dto.setSerde(toStorageDto(table.getSd(), table.getOwner()));
    dto.setAudit(new AuditDto());
    dto.setName(name);
    if (table.isSetCreateTime()) {
        dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime()));
    }
    dto.setMetadata(table.getParameters());

    final List<FieldSchema> nonPartitionColumns = table.getSd().getCols();
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final List<FieldDto> allFields =
        Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, false))
        .forEachOrdered(allFields::add);
    partitionColumns.stream()
        .map(field -> this.hiveToMetacatField(field, true))
        .forEachOrdered(allFields::add);
    dto.setFields(allFields);
    dto.setView(new ViewDto(table.getViewOriginalText(),
        table.getViewExpandedText()));
    return dto;
}
 

@Test
public void sdColsSameNumberOfElementsFullComparison() {
  left.getPartition().getSd().setCols(
      ImmutableList.of(new FieldSchema("left1", "type", "comment1"), new FieldSchema("left2", "type", "comment2")));
  List<Diff<Object, Object>> diffs = newPartitionAndMetadataComparator(FULL_COMPARISON).compare(left, right);
  assertThat(diffs, is(notNullValue()));
  assertThat(diffs.size(), is(4));
  assertThat(diffs.get(0),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getName(),
          right.getPartition().getSd().getCols().get(0).getName())));
  assertThat(diffs.get(1),
      is(newDiff(
          "Element 0 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(0).getType(),
          right.getPartition().getSd().getCols().get(0).getType())));
  assertThat(diffs.get(2),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property name of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getName(),
          right.getPartition().getSd().getCols().get(1).getName())));
  assertThat(diffs.get(3),
      is(newDiff(
          "Element 1 of collection partition.sd.cols of class com.google.common.collect.RegularImmutableList is different: Property type of class org.apache.hadoop.hive.metastore.api.FieldSchema is different",
          left.getPartition().getSd().getCols().get(1).getType(),
          right.getPartition().getSd().getCols().get(1).getType())));
}