Java Code Examples for org.apache.hadoop.hive.metastore.api.StorageDescriptor#getLocation()

The following examples show how to use org.apache.hadoop.hive.metastore.api.StorageDescriptor#getLocation() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveTableOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void finalizeGlobal(int parallelism) throws IOException {
	StorageDescriptor jobSD = hiveTablePartition.getStorageDescriptor();
	Path stagingDir = new Path(jobSD.getLocation());
	FileSystem fs = stagingDir.getFileSystem(jobConf);
	try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(new HiveConf(jobConf, HiveConf.class), hiveVersion)) {
		Table table = client.getTable(tablePath.getDatabaseName(), tablePath.getObjectName());
		if (!isDynamicPartition) {
			commitJob(stagingDir.toString());
		}
		if (isPartitioned) {
			if (isDynamicPartition) {
				FileStatus[] generatedParts = HiveStatsUtils.getFileStatusRecurse(stagingDir,
					partitionColumns.size() - hiveTablePartition.getPartitionSpec().size(), fs);
				for (FileStatus part : generatedParts) {
					commitJob(part.getPath().toString());
					LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>();
					Warehouse.makeSpecFromName(fullPartSpec, part.getPath());
					loadPartition(part.getPath(), table, fullPartSpec, client);
				}
			} else {
				LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
				for (String partCol : hiveTablePartition.getPartitionSpec().keySet()) {
					partSpec.put(partCol, hiveTablePartition.getPartitionSpec().get(partCol).toString());
				}
				loadPartition(stagingDir, table, partSpec, client);
			}
		} else {
			moveFiles(stagingDir, new Path(table.getSd().getLocation()));
		}
	} catch (TException e) {
		throw new CatalogException("Failed to query Hive metaStore", e);
	} finally {
		fs.delete(stagingDir, true);
	}
}
Example 2
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public static boolean inputPathExists(StorageDescriptor sd, JobConf job) {

    final Path path = new Path(sd.getLocation());
    try {
      // TODO: DX-16001 - make async configurable for Hive.
      final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(path, job);
      return fs.exists(path);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
Example 3
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
public static boolean inputPathExists(StorageDescriptor sd, JobConf job) {

    final Path path = new Path(sd.getLocation());
    try {
      // TODO: DX-16001 - make async configurable for Hive.
      final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(path, job);
      return fs.exists(path);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
Example 4
Source File: MetastoreAuthzBinding.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up 
private String getSdLocation(StorageDescriptor sd) {
  if (sd == null) {
    return "";
  } else {
    return sd.getLocation();
  }
}
Example 5
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static HiveStorageCapabilities getHiveStorageCapabilities(final StorageDescriptor storageDescriptor) {
  final String location = storageDescriptor.getLocation();

  if (null != location) {
    final URI uri;
    try {
      uri = URI.create(location);
    } catch (IllegalArgumentException e) {
      // unknown table source, default to HDFS.
      return HiveStorageCapabilities.DEFAULT_HDFS;
    }

    final String scheme = uri.getScheme();
    if (!Strings.isNullOrEmpty(scheme)) {
      if (scheme.regionMatches(true, 0, "s3", 0, 2)) {
        /* AWS S3 does not support impersonation, last modified times or orc split file ids. */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(false)
          .supportsLastModifiedTime(false)
          .supportsOrcSplitFileIds(false)
          .build();
      } else if (scheme.regionMatches(true, 0, "wasb", 0, 4) ||
        scheme.regionMatches(true, 0, "abfs", 0, 4) ||
        scheme.regionMatches(true, 0, "wasbs", 0, 5) ||
        scheme.regionMatches(true, 0, "abfss", 0, 5)) {
        /* DX-17365: Azure Storage does not support correct last modified times, Azure returns last modified times,
         *  however, the timestamps returned are incorrect. They reference the folder's create time rather
         *  that the folder content's last modified time. Please see Prototype.java for Azure storage fs uri schemes. */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(true)
          .supportsLastModifiedTime(false)
          .supportsOrcSplitFileIds(true)
          .build();
      } else if (!scheme.regionMatches(true, 0, "hdfs", 0, 4)) {
        /* Most hive supported non-HDFS file systems allow for impersonation and last modified times, but
           not orc split file ids.  */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(true)
          .supportsLastModifiedTime(true)
          .supportsOrcSplitFileIds(false)
          .build();
      }
    }
  }
  // Default to HDFS.
  return HiveStorageCapabilities.DEFAULT_HDFS;
}
Example 6
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
private static void addInputPath(StorageDescriptor sd, JobConf job) {
  final org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(sd.getLocation());
  FileInputFormat.addInputPath(job, path);
}
Example 7
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static HiveStorageCapabilities getHiveStorageCapabilities(final StorageDescriptor storageDescriptor) {
  final String location = storageDescriptor.getLocation();

  if (null != location) {
    final URI uri;
    try {
      uri = URI.create(location);
    } catch (IllegalArgumentException e) {
      // unknown table source, default to HDFS.
      return HiveStorageCapabilities.DEFAULT_HDFS;
    }

    final String scheme = uri.getScheme();
    if (!Strings.isNullOrEmpty(scheme)) {
      if (scheme.regionMatches(true, 0, "s3", 0, 2)) {
        /* AWS S3 does not support impersonation, last modified times or orc split file ids. */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(false)
          .supportsLastModifiedTime(false)
          .supportsOrcSplitFileIds(false)
          .build();
      } else if (scheme.regionMatches(true, 0, "wasb", 0, 4) ||
        scheme.regionMatches(true, 0, "abfs", 0, 4) ||
        scheme.regionMatches(true, 0, "wasbs", 0, 5) ||
        scheme.regionMatches(true, 0, "abfss", 0, 5)) {
        /* DX-17365: Azure Storage does not support correct last modified times, Azure returns last modified times,
         *  however, the timestamps returned are incorrect. They reference the folder's create time rather
         *  that the folder content's last modified time. Please see Prototype.java for Azure storage fs uri schemes. */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(true)
          .supportsLastModifiedTime(false)
          .supportsOrcSplitFileIds(true)
          .build();
      } else if (!scheme.regionMatches(true, 0, "hdfs", 0, 4)) {
        /* Most hive supported non-HDFS file systems allow for impersonation and last modified times, but
           not orc split file ids.  */
        return HiveStorageCapabilities.newBuilder()
          .supportsImpersonation(true)
          .supportsLastModifiedTime(true)
          .supportsOrcSplitFileIds(false)
          .build();
      }
    }
  }
  // Default to HDFS.
  return HiveStorageCapabilities.DEFAULT_HDFS;
}
Example 8
Source File: HiveMetadataUtils.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
private static void addInputPath(StorageDescriptor sd, JobConf job) {
  final org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(sd.getLocation());
  FileInputFormat.addInputPath(job, path);
}