Java Code Examples for org.apache.hadoop.hive.metastore.api.StorageDescriptor#getLocation()
The following examples show how to use
org.apache.hadoop.hive.metastore.api.StorageDescriptor#getLocation() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveTableOutputFormat.java From flink with Apache License 2.0 | 5 votes |
@Override public void finalizeGlobal(int parallelism) throws IOException { StorageDescriptor jobSD = hiveTablePartition.getStorageDescriptor(); Path stagingDir = new Path(jobSD.getLocation()); FileSystem fs = stagingDir.getFileSystem(jobConf); try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(new HiveConf(jobConf, HiveConf.class), hiveVersion)) { Table table = client.getTable(tablePath.getDatabaseName(), tablePath.getObjectName()); if (!isDynamicPartition) { commitJob(stagingDir.toString()); } if (isPartitioned) { if (isDynamicPartition) { FileStatus[] generatedParts = HiveStatsUtils.getFileStatusRecurse(stagingDir, partitionColumns.size() - hiveTablePartition.getPartitionSpec().size(), fs); for (FileStatus part : generatedParts) { commitJob(part.getPath().toString()); LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>(); Warehouse.makeSpecFromName(fullPartSpec, part.getPath()); loadPartition(part.getPath(), table, fullPartSpec, client); } } else { LinkedHashMap<String, String> partSpec = new LinkedHashMap<>(); for (String partCol : hiveTablePartition.getPartitionSpec().keySet()) { partSpec.put(partCol, hiveTablePartition.getPartitionSpec().get(partCol).toString()); } loadPartition(stagingDir, table, partSpec, client); } } else { moveFiles(stagingDir, new Path(table.getSd().getLocation())); } } catch (TException e) { throw new CatalogException("Failed to query Hive metaStore", e); } finally { fs.delete(stagingDir, true); } }
Example 2
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 5 votes |
public static boolean inputPathExists(StorageDescriptor sd, JobConf job) { final Path path = new Path(sd.getLocation()); try { // TODO: DX-16001 - make async configurable for Hive. final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(path, job); return fs.exists(path); } catch (IOException e) { throw new RuntimeException(e); } }
Example 3
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 5 votes |
public static boolean inputPathExists(StorageDescriptor sd, JobConf job) { final Path path = new Path(sd.getLocation()); try { // TODO: DX-16001 - make async configurable for Hive. final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(path, job); return fs.exists(path); } catch (IOException e) { throw new RuntimeException(e); } }
Example 4
Source File: MetastoreAuthzBinding.java From incubator-sentry with Apache License 2.0 | 5 votes |
private String getSdLocation(StorageDescriptor sd) { if (sd == null) { return ""; } else { return sd.getLocation(); } }
Example 5
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 4 votes |
public static HiveStorageCapabilities getHiveStorageCapabilities(final StorageDescriptor storageDescriptor) { final String location = storageDescriptor.getLocation(); if (null != location) { final URI uri; try { uri = URI.create(location); } catch (IllegalArgumentException e) { // unknown table source, default to HDFS. return HiveStorageCapabilities.DEFAULT_HDFS; } final String scheme = uri.getScheme(); if (!Strings.isNullOrEmpty(scheme)) { if (scheme.regionMatches(true, 0, "s3", 0, 2)) { /* AWS S3 does not support impersonation, last modified times or orc split file ids. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(false) .supportsLastModifiedTime(false) .supportsOrcSplitFileIds(false) .build(); } else if (scheme.regionMatches(true, 0, "wasb", 0, 4) || scheme.regionMatches(true, 0, "abfs", 0, 4) || scheme.regionMatches(true, 0, "wasbs", 0, 5) || scheme.regionMatches(true, 0, "abfss", 0, 5)) { /* DX-17365: Azure Storage does not support correct last modified times, Azure returns last modified times, * however, the timestamps returned are incorrect. They reference the folder's create time rather * that the folder content's last modified time. Please see Prototype.java for Azure storage fs uri schemes. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(true) .supportsLastModifiedTime(false) .supportsOrcSplitFileIds(true) .build(); } else if (!scheme.regionMatches(true, 0, "hdfs", 0, 4)) { /* Most hive supported non-HDFS file systems allow for impersonation and last modified times, but not orc split file ids. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(true) .supportsLastModifiedTime(true) .supportsOrcSplitFileIds(false) .build(); } } } // Default to HDFS. return HiveStorageCapabilities.DEFAULT_HDFS; }
Example 6
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 4 votes |
private static void addInputPath(StorageDescriptor sd, JobConf job) { final org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(sd.getLocation()); FileInputFormat.addInputPath(job, path); }
Example 7
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 4 votes |
public static HiveStorageCapabilities getHiveStorageCapabilities(final StorageDescriptor storageDescriptor) { final String location = storageDescriptor.getLocation(); if (null != location) { final URI uri; try { uri = URI.create(location); } catch (IllegalArgumentException e) { // unknown table source, default to HDFS. return HiveStorageCapabilities.DEFAULT_HDFS; } final String scheme = uri.getScheme(); if (!Strings.isNullOrEmpty(scheme)) { if (scheme.regionMatches(true, 0, "s3", 0, 2)) { /* AWS S3 does not support impersonation, last modified times or orc split file ids. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(false) .supportsLastModifiedTime(false) .supportsOrcSplitFileIds(false) .build(); } else if (scheme.regionMatches(true, 0, "wasb", 0, 4) || scheme.regionMatches(true, 0, "abfs", 0, 4) || scheme.regionMatches(true, 0, "wasbs", 0, 5) || scheme.regionMatches(true, 0, "abfss", 0, 5)) { /* DX-17365: Azure Storage does not support correct last modified times, Azure returns last modified times, * however, the timestamps returned are incorrect. They reference the folder's create time rather * that the folder content's last modified time. Please see Prototype.java for Azure storage fs uri schemes. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(true) .supportsLastModifiedTime(false) .supportsOrcSplitFileIds(true) .build(); } else if (!scheme.regionMatches(true, 0, "hdfs", 0, 4)) { /* Most hive supported non-HDFS file systems allow for impersonation and last modified times, but not orc split file ids. */ return HiveStorageCapabilities.newBuilder() .supportsImpersonation(true) .supportsLastModifiedTime(true) .supportsOrcSplitFileIds(false) .build(); } } } // Default to HDFS. return HiveStorageCapabilities.DEFAULT_HDFS; }
Example 8
Source File: HiveMetadataUtils.java From dremio-oss with Apache License 2.0 | 4 votes |
private static void addInputPath(StorageDescriptor sd, JobConf job) { final org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(sd.getLocation()); FileInputFormat.addInputPath(job, path); }