Java Code Examples for org.apache.hadoop.hive.metastore.api.StorageDescriptor#setBucketCols()
The following examples show how to use
org.apache.hadoop.hive.metastore.api.StorageDescriptor#setBucketCols() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CatalogToHiveConverter.java From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 | 6 votes |
public static StorageDescriptor convertStorageDescriptor(com.amazonaws.services.glue.model.StorageDescriptor catalogSd) { StorageDescriptor hiveSd = new StorageDescriptor(); hiveSd.setCols(convertFieldSchemaList(catalogSd.getColumns())); hiveSd.setLocation(catalogSd.getLocation()); hiveSd.setInputFormat(catalogSd.getInputFormat()); hiveSd.setOutputFormat(catalogSd.getOutputFormat()); hiveSd.setCompressed(catalogSd.getCompressed()); hiveSd.setNumBuckets(catalogSd.getNumberOfBuckets()); hiveSd.setSerdeInfo(convertSerDeInfo(catalogSd.getSerdeInfo())); hiveSd.setBucketCols(firstNonNull(catalogSd.getBucketColumns(), Lists.<String>newArrayList())); hiveSd.setSortCols(convertOrderList(catalogSd.getSortColumns())); hiveSd.setParameters(firstNonNull(catalogSd.getParameters(), Maps.<String, String>newHashMap())); hiveSd.setSkewedInfo(convertSkewedInfo(catalogSd.getSkewedInfo())); hiveSd.setStoredAsSubDirectories(catalogSd.getStoredAsSubDirectories()); return hiveSd; }
Example 2
Source File: HiveUtils.java From kite with Apache License 2.0 | 6 votes |
static Table createEmptyTable(String namespace, String name) { Table table = new Table(); table.setDbName(namespace); table.setTableName(name); table.setPartitionKeys(new ArrayList<FieldSchema>()); table.setParameters(new HashMap<String, String>()); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.setNumBuckets(-1); sd.setBucketCols(new ArrayList<String>()); sd.setCols(new ArrayList<FieldSchema>()); sd.setParameters(new HashMap<String, String>()); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); SkewedInfo skewInfo = new SkewedInfo(); skewInfo.setSkewedColNames(new ArrayList<String>()); skewInfo.setSkewedColValues(new ArrayList<List<String>>()); skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); sd.setSkewedInfo(skewInfo); table.setSd(sd); return table; }
Example 3
Source File: ThriftMetastoreUtil.java From presto with Apache License 2.0 | 5 votes |
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toImmutableList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setSkewedInfoIsSet(storage.isSkewed()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toImmutableList())); } } return sd; }
Example 4
Source File: HiveTableMetaStoreFactory.java From flink with Apache License 2.0 | 5 votes |
private void alterPartition(LinkedHashMap<String, String> partitionSpec, Path partitionPath, Partition currentPartition) throws Exception { StorageDescriptor partSD = currentPartition.getSd(); // the following logic copied from Hive::alterPartitionSpecInMemory partSD.setOutputFormat(sd.getOutputFormat()); partSD.setInputFormat(sd.getInputFormat()); partSD.getSerdeInfo().setSerializationLib(sd.getSerdeInfo().getSerializationLib()); partSD.getSerdeInfo().setParameters(sd.getSerdeInfo().getParameters()); partSD.setBucketCols(sd.getBucketCols()); partSD.setNumBuckets(sd.getNumBuckets()); partSD.setSortCols(sd.getSortCols()); partSD.setLocation(partitionPath.toString()); client.alter_partition(database, tableName, currentPartition); }
Example 5
Source File: HiveMetaStoreUtils.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private static StorageDescriptor getStorageDescriptor(HiveRegistrationUnit unit) { State props = unit.getStorageProps(); StorageDescriptor sd = new StorageDescriptor(); sd.setParameters(getParameters(props)); //Treat AVRO and other formats differently. Details can be found in GOBBLIN-877 if (unit.isRegisterSchema() || (unit.getInputFormat().isPresent() && !unit.getInputFormat().get().equals(AvroContainerInputFormat.class.getName()))) { sd.setCols(getFieldSchemas(unit)); } if (unit.getLocation().isPresent()) { sd.setLocation(unit.getLocation().get()); } if (unit.getInputFormat().isPresent()) { sd.setInputFormat(unit.getInputFormat().get()); } if (unit.getOutputFormat().isPresent()) { sd.setOutputFormat(unit.getOutputFormat().get()); } if (unit.getIsCompressed().isPresent()) { sd.setCompressed(unit.getIsCompressed().get()); } if (unit.getNumBuckets().isPresent()) { sd.setNumBuckets(unit.getNumBuckets().get()); } if (unit.getBucketColumns().isPresent()) { sd.setBucketCols(unit.getBucketColumns().get()); } if (unit.getIsStoredAsSubDirs().isPresent()) { sd.setStoredAsSubDirectories(unit.getIsStoredAsSubDirs().get()); } sd.setSerdeInfo(getSerDeInfo(unit)); return sd; }
Example 6
Source File: HiveConvertersImpl.java From metacat with Apache License 2.0 | 4 votes |
private StorageDescriptor fromStorageDto(@Nullable final StorageDto storageDto, @Nullable final String serdeName) { // // Set all required fields to null. This is to simulate Hive behavior. // Setting it to empty string failed certain hive operations. // final StorageDescriptor result = new StorageDescriptor(); String inputFormat = null; String location = null; String outputFormat = null; String serializationLib = null; Map<String, String> sdParams = Maps.newHashMap(); Map<String, String> serdeParams = Maps.newHashMap(); if (storageDto != null) { if (storageDto.getInputFormat() != null) { inputFormat = storageDto.getInputFormat(); } if (storageDto.getUri() != null) { location = storageDto.getUri(); } if (storageDto.getOutputFormat() != null) { outputFormat = storageDto.getOutputFormat(); } if (storageDto.getSerializationLib() != null) { serializationLib = storageDto.getSerializationLib(); } if (storageDto.getParameters() != null) { sdParams = storageDto.getParameters(); } if (storageDto.getSerdeInfoParameters() != null) { serdeParams = storageDto.getSerdeInfoParameters(); } } result.setSerdeInfo(new SerDeInfo(serdeName, serializationLib, serdeParams)); result.setBucketCols(Collections.emptyList()); result.setSortCols(Collections.emptyList()); result.setInputFormat(inputFormat); result.setLocation(location); result.setOutputFormat(outputFormat); result.setCols(Collections.emptyList()); // Setting an empty skewed info. result.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); result.setParameters(sdParams); return result; }