Java Code Examples for org.apache.hadoop.fs.FileSystem#getUri()

The following examples show how to use org.apache.hadoop.fs.FileSystem#getUri() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HadoopClientServicesImpl.java    From pentaho-hadoop-shims with Apache License 2.0 6 votes vote down vote up
public HadoopFileSystem getFileSystem( NamedCluster namedCluster, URI uri ) throws IOException {
  final Configuration configuration = hadoopShim.createConfiguration( namedCluster );
  FileSystem fileSystem = (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
  if ( fileSystem instanceof LocalFileSystem ) {
    LOGGER.error( "Got a local filesystem, was expecting an hdfs connection" );
    throw new IOException( "Got a local filesystem, was expecting an hdfs connection" );
  }

  final URI finalUri = fileSystem.getUri() != null ? fileSystem.getUri() : uri;
  HadoopFileSystem hadoopFileSystem = new HadoopFileSystemImpl( () -> {
    try {
      return finalUri != null
        ? (FileSystem) hadoopShim.getFileSystem( finalUri, configuration, (NamedCluster) namedCluster ).getDelegate()
        : (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
    } catch ( IOException | InterruptedException e ) {
      LOGGER.debug( "Error looking up/creating the file system ", e );
      return null;
    }
  } );
  ( (HadoopFileSystemImpl) hadoopFileSystem ).setNamedCluster( namedCluster );

  return hadoopFileSystem;
}
 
Example 2
Source File: HadoopSecondaryFileSystemConfigurationTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/**
 * Executes before each test.
 * @throws Exception If failed.
 */
private void before() throws Exception {
    initSecondary();

    if (passPrimaryConfiguration) {
        Configuration primaryFsCfg = configuration(primaryCfgScheme, primaryCfgAuthority, skipEmbed, skipLocShmem);

        primaryConfFullPath = writeConfiguration(primaryFsCfg, PRIMARY_CFG_PATH);
    }
    else
        primaryConfFullPath = null;

    CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();

    fac.setConfigPaths(primaryConfFullPath);
    fac.setUri(primaryFsUriStr);

    HadoopFileSystemFactoryDelegate facDelegate = HadoopDelegateUtils.fileSystemFactoryDelegate(
        getClass().getClassLoader(), fac);

    facDelegate.start();

    primaryFs = (FileSystem)facDelegate.get(null); //provider.createFileSystem(null);

    primaryFsUri = primaryFs.getUri();
}
 
Example 3
Source File: SnapshotDescriptionUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Commits the snapshot process by moving the working snapshot
 * to the finalized filepath
 *
 * @param snapshotDir The file path of the completed snapshots
 * @param workingDir  The file path of the in progress snapshots
 * @param fs The file system of the completed snapshots
 * @param workingDirFs The file system of the in progress snapshots
 * @param conf Configuration
 *
 * @throws SnapshotCreationException if the snapshot could not be moved
 * @throws IOException the filesystem could not be reached
 */
public static void completeSnapshot(Path snapshotDir, Path workingDir, FileSystem fs,
  FileSystem workingDirFs, final Configuration conf)
  throws SnapshotCreationException, IOException {
  LOG.debug("Sentinel is done, just moving the snapshot from " + workingDir + " to "
    + snapshotDir);
  // If the working and completed snapshot directory are on the same file system, attempt
  // to rename the working snapshot directory to the completed location. If that fails,
  // or the file systems differ, attempt to copy the directory over, throwing an exception
  // if this fails
  URI workingURI = workingDirFs.getUri();
  URI rootURI = fs.getUri();
  if ((!workingURI.getScheme().equals(rootURI.getScheme()) ||
    workingURI.getAuthority() == null ||
    !workingURI.getAuthority().equals(rootURI.getAuthority()) ||
    workingURI.getUserInfo() == null ||
    !workingURI.getUserInfo().equals(rootURI.getUserInfo()) ||
    !fs.rename(workingDir, snapshotDir)) && !FileUtil.copy(workingDirFs, workingDir, fs,
    snapshotDir, true, true, conf)) {
    throw new SnapshotCreationException("Failed to copy working directory(" + workingDir
      + ") to completed directory(" + snapshotDir + ").");
  }
}
 
Example 4
Source File: HadoopFileSystemFactoryImpl.java    From pentaho-hadoop-shims with Apache License 2.0 6 votes vote down vote up
@Override
public HadoopFileSystem create( NamedCluster namedCluster, URI uri ) throws IOException {
  final Configuration configuration = hadoopShim.createConfiguration( namedCluster );
  FileSystem fileSystem = (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
  if ( fileSystem instanceof LocalFileSystem ) {
    LOGGER.error( "Got a local filesystem, was expecting an hdfs connection" );
    throw new IOException( "Got a local filesystem, was expecting an hdfs connection" );
  }

  final URI finalUri = fileSystem.getUri() != null ? fileSystem.getUri() : uri;
  HadoopFileSystem hadoopFileSystem = new HadoopFileSystemImpl( () -> {
    try {
      return finalUri != null
        ? (FileSystem) hadoopShim.getFileSystem( finalUri, configuration, (NamedCluster) namedCluster ).getDelegate()
        : (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
    } catch ( IOException | InterruptedException e ) {
      LOGGER.debug( "Error looking up/creating the file system ", e );
      return null;
    }
  } );
  ( (HadoopFileSystemImpl) hadoopFileSystem ).setNamedCluster( namedCluster );

  return hadoopFileSystem;
}
 
Example 5
Source File: ProxiedFileSystemCache.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private static URI resolveUri(URI uri, Configuration configuration, FileSystem fileSystem) throws IOException {
  if (uri != null) {
    return uri;
  }
  if (fileSystem != null) {
    return fileSystem.getUri();
  }
  if (configuration != null) {
    return FileSystem.getDefaultUri(configuration);
  }
  throw new IOException("FileSystem URI could not be determined from available inputs.");
}
 
Example 6
Source File: GobblinClusterUtils.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
public static Path getAppWorkDirPathFromConfig(Config config, FileSystem fs,
    String applicationName, String applicationId) {
  if (config.hasPath(GobblinClusterConfigurationKeys.CLUSTER_WORK_DIR)) {
    return new Path(new Path(fs.getUri()), PathUtils.combinePaths(config.getString(GobblinClusterConfigurationKeys.CLUSTER_WORK_DIR),
        getAppWorkDirPath(applicationName, applicationId)));
  }
  return new Path(fs.getHomeDirectory(), getAppWorkDirPath(applicationName, applicationId));
}
 
Example 7
Source File: TestRaidShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private static DistributedFileSystem getDFS(
      Configuration conf, FileSystem dfs) throws IOException {
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  FileSystem.closeAll();
  return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf);
}
 
Example 8
Source File: JarManager.java    From spork with Apache License 2.0 5 votes vote down vote up
/**
 * Add the qualified path name of jars containing the given classes
 *
 * @param fs
 *            FileSystem object
 * @param jars
 *            the resolved path names to be added to this set
 * @param classes
 *            classes to find
 */
private static void addQualifiedJarsName(FileSystem fs, Set<String> jars, Class<?>... classes) {
    URI fsUri = fs.getUri();
    Path workingDir = fs.getWorkingDirectory();
    for (Class<?> clazz : classes) {
        String jarName = findContainingJar(clazz);
        if (jarName == null) {
            log.warn("Could not find jar for class " + clazz);
            continue;
        }
        jars.add(new Path(jarName).makeQualified(fsUri, workingDir).toString());
    }
}
 
Example 9
Source File: AdminHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up
static DistributedFileSystem getDFS(Configuration conf)
    throws IOException {
  FileSystem fs = FileSystem.get(conf);
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() +
        " is not an HDFS file system");
  }
  return (DistributedFileSystem)fs;
}
 
Example 10
Source File: JobResourceUploader.java    From big-c with Apache License 2.0 5 votes vote down vote up
private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch (UnknownHostException ue) {
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  } else if (srcHost == null && dstHost != null) {
    return false;
  } else if (srcHost != null && dstHost == null) {
    return false;
  }
  // check for ports
  if (srcUri.getPort() != dstUri.getPort()) {
    return false;
  }
  return true;
}
 
Example 11
Source File: DistCpUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch(UnknownHostException ue) {
      if (LOG.isDebugEnabled())
        LOG.debug("Could not compare file-systems. Unknown host: ", ue);
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  }
  else if (srcHost == null && dstHost != null) {
    return false;
  }
  else if (srcHost != null) {
    return false;
  }

  //check for ports

  return srcUri.getPort() == dstUri.getPort();
}
 
Example 12
Source File: DistCpUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Determines if a file system supports XAttrs by running a getXAttrs request
 * on the file system root. This method is used before distcp job submission
 * to fail fast if the user requested preserving XAttrs, but the file system
 * cannot support XAttrs.
 * 
 * @param fs FileSystem to check
 * @throws XAttrsNotSupportedException if fs does not support XAttrs
 */
public static void checkFileSystemXAttrSupport(FileSystem fs)
    throws XAttrsNotSupportedException {
  try {
    fs.getXAttrs(new Path(Path.SEPARATOR));
  } catch (Exception e) {
    throw new XAttrsNotSupportedException("XAttrs not supported for file system: "
      + fs.getUri());
  }
}
 
Example 13
Source File: JobClient.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch(UnknownHostException ue) {
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  }
  else if (srcHost == null && dstHost != null) {
    return false;
  }
  else if (srcHost != null && dstHost == null) {
    return false;
  }
  //check for ports
  if (srcUri.getPort() != dstUri.getPort()) {
    return false;
  }
  return true;
}
 
Example 14
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected DistributedFileSystem getDFS() throws IOException {
  FileSystem fs = getFS();
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
    " is not an HDFS file system");
  }
  return (DistributedFileSystem)fs;
}
 
Example 15
Source File: DriverTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Test
public void testDriverAddSingleRecordToExistingRow() throws Exception {
  FileSystem fileSystem = miniCluster.getFileSystem();
  Path root = new Path(fileSystem.getUri() + "/");

  String tableName = "testDriverAddSingleRecordToExistingRow";
  Iface client = getClient();
  creatTable(tableName, new Path(root, "tables"), true);
  addRow(client, tableName, "row1", "record1", "value1");

  Driver driver = new Driver();
  driver.setConf(conf);

  String mrIncWorkingPathStr = new Path(root, "working").toString();
  generateData(mrIncWorkingPathStr);
  String outputPathStr = new Path(root, "output").toString();
  String blurZkConnection = miniCluster.getZkConnectionString();

  assertEquals(0, driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" }));

  client.loadData(tableName, outputPathStr);

  waitUntilAllImportsAreCompleted(client, tableName);

  TableStats tableStats = client.tableStats(tableName);
  assertEquals(1, tableStats.getRowCount());
  assertEquals(2, tableStats.getRecordCount());
}
 
Example 16
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Constructor */
public DFSAdminCommand(FileSystem fs) {
  super(fs.getConf());
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
        " is not a distributed file system");
  }
  this.dfs = (DistributedFileSystem)fs;
}
 
Example 17
Source File: TestUrlStreamHandler.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
/**
 * Test opening and reading from an InputStream through a hdfs:// URL.
 * <p>
 * First generate a file with some content through the FileSystem API, then
 * try to open and read the file through the URL stream API.
 * 
 * @throws IOException
 */
public void testDfsUrls() throws IOException {

  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  FileSystem fs = cluster.getFileSystem();

  // Setup our own factory
  // setURLSteramHandlerFactor is can be set at most once in the JVM
  // the new URLStreamHandler is valid for all tests cases 
  // in TestStreamHandler
  FsUrlStreamHandlerFactory factory =
      new org.apache.hadoop.fs.FsUrlStreamHandlerFactory();
  java.net.URL.setURLStreamHandlerFactory(factory);

  Path filePath = new Path("/thefile");

  try {
    byte[] fileContent = new byte[1024];
    for (int i = 0; i < fileContent.length; ++i)
      fileContent[i] = (byte) i;

    // First create the file through the FileSystem API
    OutputStream os = fs.create(filePath);
    os.write(fileContent);
    os.close();

    // Second, open and read the file content through the URL API
    URI uri = fs.getUri();
    URL fileURL =
        new URL(uri.getScheme(), uri.getHost(), uri.getPort(), filePath
            .toString());

    InputStream is = fileURL.openStream();
    assertNotNull(is);

    byte[] bytes = new byte[4096];
    assertEquals(1024, is.read(bytes));
    is.close();

    for (int i = 0; i < fileContent.length; ++i)
      assertEquals(fileContent[i], bytes[i]);

    // Cleanup: delete the file
    fs.delete(filePath, false);

  } finally {
    fs.close();
    cluster.shutdown();
  }

}
 
Example 18
Source File: TestHBaseOnOtherDfsCluster.java    From hbase with Apache License 2.0 4 votes vote down vote up
private void assertFsSameUri(FileSystem sourceFs, FileSystem targetFs) {
  Path source = new Path(sourceFs.getUri());
  Path target = new Path(targetFs.getUri());
  assertEquals(source, target);
}
 
Example 19
Source File: FSUtils.java    From hudi with Apache License 2.0 4 votes vote down vote up
/**
 * Get DFS full partition path (e.g. hdfs://ip-address:8020:/<absolute path>)
 */
public static String getDFSFullPartitionPath(FileSystem fs, Path partitionPath) {
  return fs.getUri() + partitionPath.toUri().getRawPath();
}
 
Example 20
Source File: ConfigBasedDatasetTest.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
public Collection<? extends CopyEntity> testGetCopyableFilesHelper(String sourceDir, String destinationDir,
    long sourceWatermark, boolean isFilterEnabled) throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  URI local = localFs.getUri();

  Properties properties = new Properties();
  properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
  PathFilter pathFilter = DatasetUtils.instantiatePathFilter(properties);
  boolean applyFilterToDirectories = false;
  if (isFilterEnabled) {
    properties.setProperty(DatasetUtils.CONFIGURATION_KEY_PREFIX + "path.filter.class",
        "org.apache.gobblin.util.filters.HiddenFilter");
    properties.setProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES, "true");

    pathFilter = DatasetUtils.instantiatePathFilter(properties);
    applyFilterToDirectories =
        Boolean.parseBoolean(properties.getProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES, "false"));
  }

  CopyConfiguration copyConfiguration =
      CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties)
          .publishDir(new Path(destinationDir))
          .preserve(PreserveAttributes.fromMnemonicString("ugp"))
          .build();

  ReplicationMetaData mockMetaData = Mockito.mock(ReplicationMetaData.class);
  Mockito.when(mockMetaData.toString()).thenReturn("Mock Meta Data");

  ReplicationConfiguration mockRC = Mockito.mock(ReplicationConfiguration.class);
  Mockito.when(mockRC.getCopyMode()).thenReturn(ReplicationCopyMode.PULL);
  Mockito.when(mockRC.getMetaData()).thenReturn(mockMetaData);
  Mockito.when(mockRC.getVersionStrategyFromConfigStore()).thenReturn(Optional.of(DataFileVersionStrategy.DEFAULT_DATA_FILE_VERSION_STRATEGY));
  Mockito.when(mockRC.getEnforceFileSizeMatchFromConfigStore()).thenReturn(Optional.absent());
  HadoopFsEndPoint copyFrom = Mockito.mock(HadoopFsEndPoint.class);
  Mockito.when(copyFrom.getDatasetPath()).thenReturn(new Path(sourceDir));
  Mockito.when(copyFrom.getFsURI()).thenReturn(local);
  ComparableWatermark sw = new LongWatermark(sourceWatermark);
  Mockito.when(copyFrom.getWatermark()).thenReturn(Optional.of(sw));
  Mockito.when(copyFrom.getFiles())
      .thenReturn(
          FileListUtils.listFilesRecursively(localFs, new Path(sourceDir), pathFilter, applyFilterToDirectories));

  HadoopFsEndPoint copyTo = Mockito.mock(HadoopFsEndPoint.class);
  Mockito.when(copyTo.getDatasetPath()).thenReturn(new Path(destinationDir));
  Mockito.when(copyTo.getFsURI()).thenReturn(local);
  Optional<ComparableWatermark> tmp = Optional.absent();
  Mockito.when(copyTo.getWatermark()).thenReturn(tmp);
  Mockito.when(copyTo.getFiles())
      .thenReturn(FileListUtils.listFilesRecursively(localFs, new Path(destinationDir), pathFilter,
          applyFilterToDirectories));

  CopyRoute route = Mockito.mock(CopyRoute.class);
  Mockito.when(route.getCopyFrom()).thenReturn(copyFrom);
  Mockito.when(route.getCopyTo()).thenReturn(copyTo);

  ConfigBasedDataset dataset = new ConfigBasedDataset(mockRC, properties, route);
  Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(localFs, copyConfiguration);
  return copyableFiles;
}