Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#getValues()

private static boolean partitionMatches(Partition partition, String databaseName, String tableName, List<String> parts)
{
    if (!partition.getDbName().equals(databaseName) ||
            !partition.getTableName().equals(tableName)) {
        return false;
    }
    List<String> values = partition.getValues();
    if (values.size() != parts.size()) {
        return false;
    }
    for (int i = 0; i < values.size(); i++) {
        String part = parts.get(i);
        if (!part.isEmpty() && !values.get(i).equals(part)) {
            return false;
        }
    }
    return true;
}
 

public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enforceVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enforceVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 

public static List<PartitionValue> getPartitionValues(Table table, Partition partition, boolean enableVarcharWidth) {
  if (partition == null) {
    return Collections.emptyList();
  }

  final List<String> partitionValues = partition.getValues();
  final List<PartitionValue> output = new ArrayList<>();
  final List<FieldSchema> partitionKeys = table.getPartitionKeys();
  for (int i = 0; i < partitionKeys.size(); i++) {
    final PartitionValue value = getPartitionValue(partitionKeys.get(i), partitionValues.get(i), enableVarcharWidth);
    if (value != null) {
      output.add(value);
    }
  }
  return output;
}
 

private void dropAndDeletePartitions(CloseableMetaStoreClient client, Predicate<String> shouldDelete)
  throws MetaException, TException, NoSuchObjectException {
  Table replicaTable = client.getTable(databaseName, tableName);
  List<FieldSchema> partitionKeys = replicaTable.getPartitionKeys();
  if (partitionKeys == null || partitionKeys.isEmpty()) {
    // unpartitioned table nothing to delete
    return;
  }
  PartitionIterator partitionIterator = new PartitionIterator(client, replicaTable, (short) 1000);
  while (partitionIterator.hasNext()) {
    Partition replicaPartition = partitionIterator.next();
    List<String> values = replicaPartition.getValues();
    String partitionName = Warehouse.makePartName(partitionKeys, values);
    if (shouldDelete.apply(partitionName)) {
      log
          .info("Dropping partition for replica table: "
              + databaseName
              + "."
              + tableName
              + ", partition value: '"
              + partitionName
              + "'");
      client.dropPartition(databaseName, tableName, partitionName, DELETE_DATA);
      Path oldLocation = locationAsPath(replicaPartition);
      String oldEventId = replicaPartition.getParameters().get(REPLICATION_EVENT.parameterName());
      cleanupLocationManager.addCleanupLocation(oldEventId, oldLocation);
    }
  }
}
 

private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 

/**
 * Iterate over the storage partitions and find if there are any new partitions that need to be added or updated.
 * Generate a list of PartitionEvent based on the changes required.
 */
List<PartitionEvent> getPartitionEvents(List<Partition> tablePartitions, List<String> partitionStoragePartitions) {
  Map<String, String> paths = new HashMap<>();
  for (Partition tablePartition : tablePartitions) {
    List<String> hivePartitionValues = tablePartition.getValues();
    Collections.sort(hivePartitionValues);
    String fullTablePartitionPath =
        Path.getPathWithoutSchemeAndAuthority(new Path(tablePartition.getSd().getLocation())).toUri().getPath();
    paths.put(String.join(", ", hivePartitionValues), fullTablePartitionPath);
  }

  List<PartitionEvent> events = new ArrayList<>();
  for (String storagePartition : partitionStoragePartitions) {
    Path storagePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition);
    String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
    // Check if the partition values or if hdfs path is the same
    List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
    Collections.sort(storagePartitionValues);
    if (!storagePartitionValues.isEmpty()) {
      String storageValue = String.join(", ", storagePartitionValues);
      if (!paths.containsKey(storageValue)) {
        events.add(PartitionEvent.newPartitionAddEvent(storagePartition));
      } else if (!paths.get(storageValue).equals(fullStoragePartitionPath)) {
        events.add(PartitionEvent.newPartitionUpdateEvent(storagePartition));
      }
    }
  }
  return events;
}
 

private ReaderContext getReaderContext(Read readRequest, Integer partitionIndexToRead)
    throws Exception {
  final List<Partition> partitions =
      metaStoreClient.listPartitions(
          readRequest.getDatabase(), readRequest.getTable(), Short.MAX_VALUE);
  final Partition partition = partitions.get(partitionIndexToRead);
  checkArgument(
      partition != null, "Unable to find a partition to read at index " + partitionIndexToRead);

  final int desiredSplitCount = HCatalogUtils.getSplitCount(readRequest, partition);
  final List<String> values = partition.getValues();
  final List<String> partitionCols = readRequest.getPartitionCols();
  checkArgument(
      values.size() == partitionCols.size(),
      "Number of input partitions should be equal to the values of list partition values.");

  List<String> filter = new ArrayList<>();
  for (int i = 0; i < partitionCols.size(); i++) {
    filter.add(partitionCols.get(i) + "=" + "'" + values.get(i) + "'");
  }
  final String filterString = String.join(" and ", filter);

  ReadEntity entity =
      new ReadEntity.Builder()
          .withDatabase(readRequest.getDatabase())
          .withFilter(filterString)
          .withTable(readRequest.getTable())
          .build();
  // pass the 'desired' split count as an hint to the API
  Map<String, String> configProps = new HashMap<>(readRequest.getConfigProperties());
  configProps.put(
      HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, String.valueOf(desiredSplitCount));
  return DataTransferFactory.getHCatReader(entity, configProps).prepareRead();
}
 

private String partitionToString(Partition partition) {
  return "Partition values: " + partition.getValues();
}
 

public static String getPartitionValueLogString(Partition partition) {
  return ((null == partition) || (null == partition.getValues()) ? "default" :
    HiveMetadataUtils.PARTITION_FIELD_SPLIT_KEY_JOINER.join(partition.getValues()));
}
 

public static String getPartitionValueLogString(Partition partition) {
  return ((null == partition) || (null == partition.getValues()) ? "default" :
    HiveMetadataUtils.PARTITION_FIELD_SPLIT_KEY_JOINER.join(partition.getValues()));
}
 

private void monitorAndForwardSplits(
		SourceContext<TimestampedHiveInputSplit> context) throws Exception {
	assert (Thread.holdsLock(checkpointLock));

	List<Tuple2<Partition, Long>> partitions = fetcher.fetchPartitions(this.context, currentReadTime);

	if (partitions.isEmpty()) {
		return;
	}

	partitions.sort((o1, o2) -> (int) (o1.f1 - o2.f1));

	long maxTimestamp = Long.MIN_VALUE;
	Set<List<String>> nextDistinctParts = new HashSet<>();
	for (Tuple2<Partition, Long> tuple2 : partitions) {
		Partition partition = tuple2.f0;
		List<String> partSpec = partition.getValues();
		if (!this.distinctPartitions.contains(partSpec)) {
			this.distinctPartitions.add(partSpec);
			long timestamp = tuple2.f1;
			if (timestamp > currentReadTime) {
				nextDistinctParts.add(partSpec);
			}
			if (timestamp > maxTimestamp) {
				maxTimestamp = timestamp;
			}
			LOG.info("Found new partition {} of table {}, forwarding splits to downstream readers",
					partSpec, tablePath.getFullName());
			HiveTableInputSplit[] splits = HiveTableInputFormat.createInputSplits(
					this.readerParallelism,
					Collections.singletonList(toHiveTablePartition(partition)),
					this.conf.conf());
			for (HiveTableInputSplit split : splits) {
				context.collect(new TimestampedHiveInputSplit(timestamp, split));
			}
		}
	}

	if (maxTimestamp > currentReadTime) {
		currentReadTime = maxTimestamp;
		distinctPartitions.clear();
		distinctPartitions.addAll(nextDistinctParts);
	}
}