Java Code Examples for org.apache.hadoop.io.nativeio.NativeIO#isAvailable()

The following examples show how to use org.apache.hadoop.io.nativeio.NativeIO#isAvailable() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RawLocalFileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private LocalFSFileOutputStream(Path f, boolean append,
    FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
Example 2
Source File: RawLocalFileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
    throws IOException {
  if (permission == null) {
    return p2f.mkdir();
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      try {
        NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
        return true;
      } catch (IOException e) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format(
              "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
              p2f, permission.toShort()), e);
        }
        return false;
      }
    } else {
      boolean b = p2f.mkdir();
      if (b) {
        setPermission(p, permission);
      }
      return b;
    }
  }
}
Example 3
Source File: RawLocalFileSystem.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
    throws IOException {
  if (permission == null) {
    permission = FsPermission.getDirDefault();
  }
  permission = permission.applyUMask(FsPermission.getUMask(getConf()));
  if (Shell.WINDOWS && NativeIO.isAvailable()) {
    try {
      NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
      return true;
    } catch (IOException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(
            "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
            p2f, permission.toShort()), e);
      }
      return false;
    }
  } else {
    boolean b = p2f.mkdir();
    if (b) {
      setPermission(p, permission);
    }
    return b;
  }
}
Example 4
Source File: RawLocalFileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private LocalFSFileOutputStream(Path f, boolean append,
    FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
Example 5
Source File: RawLocalFileSystem.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
/**
 * Use the command chmod to set permission.
 */
@Override
public void setPermission(Path p, FsPermission permission)
    throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
        permission.toShort());
  } else {
    Files.setPosixFilePermissions(Paths.get(p.toUri()),
        PosixFilePermissions.fromString(permission.toString()));
  }
}
Example 6
Source File: ShortCircuitShm.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Create the ShortCircuitShm.
 * 
 * @param shmId       The ID to use.
 * @param stream      The stream that we're going to use to create this 
 *                    shared memory segment.
 *                    
 *                    Although this is a FileInputStream, we are going to
 *                    assume that the underlying file descriptor is writable
 *                    as well as readable. It would be more appropriate to use
 *                    a RandomAccessFile here, but that class does not have
 *                    any public accessor which returns a FileDescriptor,
 *                    unlike FileInputStream.
 */
public ShortCircuitShm(ShmId shmId, FileInputStream stream)
      throws IOException {
  if (!NativeIO.isAvailable()) {
    throw new UnsupportedOperationException("NativeIO is not available.");
  }
  if (Shell.WINDOWS) {
    throw new UnsupportedOperationException(
        "DfsClientShm is not yet implemented for Windows.");
  }
  if (unsafe == null) {
    throw new UnsupportedOperationException(
        "can't use DfsClientShm because we failed to " +
        "load misc.Unsafe.");
  }
  this.shmId = shmId;
  this.mmappedLength = getUsableLength(stream);
  this.baseAddress = POSIX.mmap(stream.getFD(), 
      POSIX.MMAP_PROT_READ | POSIX.MMAP_PROT_WRITE, true, mmappedLength);
  this.slots = new Slot[mmappedLength / BYTES_PER_SLOT];
  this.allocatedSlots = new BitSet(slots.length);
  if (LOG.isTraceEnabled()) {
    LOG.trace("creating " + this.getClass().getSimpleName() +
        "(shmId=" + shmId +
        ", mmappedLength=" + mmappedLength +
        ", baseAddress=" + String.format("%x", baseAddress) +
        ", slots.length=" + slots.length + ")");
  }
}
Example 7
Source File: RawLocalFileSystem.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
/**
 * Load file permission information (UNIX symbol rwxrwxrwx, sticky bit info).
 *
 * To improve peformance, give priority to native stat() call. First try get
 * permission information by using native JNI call then fall back to use non
 * native (ProcessBuilder) call in case native lib is not loaded or native
 * call is not successful
 */
private synchronized void loadPermissionInfo() {
  if (!isPermissionLoaded() && NativeIO.isAvailable()) {
    try {
      loadPermissionInfoByNativeIO();
    } catch (IOException ex) {
      LOG.debug("Native call failed", ex);
    }
  }

  if (!isPermissionLoaded()) {
    loadPermissionInfoByNonNativeIO();
  }
}
Example 8
Source File: RawLocalFileSystem.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
private LocalFSFileOutputStream(Path f, boolean append,
                                FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (!append && permission == null) {
    permission = FsPermission.getFileDefault();
  }
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    permission = permission.applyUMask(FsPermission.getUMask(getConf()));
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
Example 9
Source File: FileUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
private static void execSetPermission(File f,
                                      FsPermission permission
)  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
  } else {
    execCommand(f, Shell.getSetPermissionCommand(
        String.format("%04o", permission.toShort()), false));
  }
}
Example 10
Source File: FileUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
/**
 * Set permissions to the required value. Uses the java primitives instead
 * of forking if group == other.
 * @param f the file to change
 * @param permission the new permissions
 * @throws IOException exception on setPermission
 */
public static void setPermission(File f, FsPermission permission
) throws IOException {
  FsAction user = permission.getUserAction();
  FsAction group = permission.getGroupAction();
  FsAction other = permission.getOtherAction();

  // use the native/fork if the group/other permissions are different
  // or if the native is available or on Windows
  if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
    execSetPermission(f, permission);
    return;
  }

  boolean rv = true;

  // read perms
  rv = f.setReadable(group.implies(FsAction.READ), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
    rv = f.setReadable(user.implies(FsAction.READ), true);
    checkReturnValue(rv, f, permission);
  }

  // write perms
  rv = f.setWritable(group.implies(FsAction.WRITE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
    rv = f.setWritable(user.implies(FsAction.WRITE), true);
    checkReturnValue(rv, f, permission);
  }

  // exec perms
  rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
    rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
    checkReturnValue(rv, f, permission);
  }
}
Example 11
Source File: ReadaheadPool.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Return the singleton instance for the current process.
 */
public static ReadaheadPool getInstance() {
  synchronized (ReadaheadPool.class) {
    if (instance == null && NativeIO.isAvailable()) {
      instance = new ReadaheadPool();
    }
    return instance;
  }
}
Example 12
Source File: RawLocalFileSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Use the command chmod to set permission.
 */
@Override
public void setPermission(Path p, FsPermission permission)
  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                   permission.toShort());
  } else {
    String perm = String.format("%04o", permission.toShort());
    Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
      FileUtil.makeShellPath(pathToFile(p), true)));
  }
}
Example 13
Source File: ReadaheadPool.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Return the singleton instance for the current process.
 */
public static ReadaheadPool getInstance() {
  synchronized (ReadaheadPool.class) {
    if (instance == null && NativeIO.isAvailable()) {
      instance = new ReadaheadPool();
    }
    return instance;
  }
}
Example 14
Source File: FileUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static void execSetPermission(File f, 
                                      FsPermission permission
                                     )  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
  } else {
    execCommand(f, Shell.getSetPermissionCommand(
                String.format("%04o", permission.toShort()), false));
  }
}
Example 15
Source File: FileUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Set permissions to the required value. Uses the java primitives instead
 * of forking if group == other.
 * @param f the file to change
 * @param permission the new permissions
 * @throws IOException
 */
public static void setPermission(File f, FsPermission permission
                                 ) throws IOException {
  FsAction user = permission.getUserAction();
  FsAction group = permission.getGroupAction();
  FsAction other = permission.getOtherAction();

  // use the native/fork if the group/other permissions are different
  // or if the native is available or on Windows
  if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
    execSetPermission(f, permission);
    return;
  }
  
  boolean rv = true;
  
  // read perms
  rv = f.setReadable(group.implies(FsAction.READ), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
    rv = f.setReadable(user.implies(FsAction.READ), true);
    checkReturnValue(rv, f, permission);
  }

  // write perms
  rv = f.setWritable(group.implies(FsAction.WRITE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
    rv = f.setWritable(user.implies(FsAction.WRITE), true);
    checkReturnValue(rv, f, permission);
  }

  // exec perms
  rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
    rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
    checkReturnValue(rv, f, permission);
  }
}
Example 16
Source File: ShortCircuitShm.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Create the ShortCircuitShm.
 * 
 * @param shmId       The ID to use.
 * @param stream      The stream that we're going to use to create this 
 *                    shared memory segment.
 *                    
 *                    Although this is a FileInputStream, we are going to
 *                    assume that the underlying file descriptor is writable
 *                    as well as readable. It would be more appropriate to use
 *                    a RandomAccessFile here, but that class does not have
 *                    any public accessor which returns a FileDescriptor,
 *                    unlike FileInputStream.
 */
public ShortCircuitShm(ShmId shmId, FileInputStream stream)
      throws IOException {
  if (!NativeIO.isAvailable()) {
    throw new UnsupportedOperationException("NativeIO is not available.");
  }
  if (Shell.WINDOWS) {
    throw new UnsupportedOperationException(
        "DfsClientShm is not yet implemented for Windows.");
  }
  if (unsafe == null) {
    throw new UnsupportedOperationException(
        "can't use DfsClientShm because we failed to " +
        "load misc.Unsafe.");
  }
  this.shmId = shmId;
  this.mmappedLength = getUsableLength(stream);
  this.baseAddress = POSIX.mmap(stream.getFD(), 
      POSIX.MMAP_PROT_READ | POSIX.MMAP_PROT_WRITE, true, mmappedLength);
  this.slots = new Slot[mmappedLength / BYTES_PER_SLOT];
  this.allocatedSlots = new BitSet(slots.length);
  if (LOG.isTraceEnabled()) {
    LOG.trace("creating " + this.getClass().getSimpleName() +
        "(shmId=" + shmId +
        ", mmappedLength=" + mmappedLength +
        ", baseAddress=" + String.format("%x", baseAddress) +
        ", slots.length=" + slots.length + ")");
  }
}
Example 17
Source File: ReadaheadPool.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Return the singleton instance for the current process.
 */
public static ReadaheadPool getInstance() {
  synchronized (ReadaheadPool.class) {
    if (instance == null && NativeIO.isAvailable()) {
      instance = new ReadaheadPool();
    }
    return instance;
  }
}
Example 18
Source File: RawLocalFileSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Use the command chmod to set permission.
 */
@Override
public void setPermission(Path p, FsPermission permission)
  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                   permission.toShort());
  } else {
    String perm = String.format("%04o", permission.toShort());
    Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
      FileUtil.makeShellPath(pathToFile(p), true)));
  }
}
Example 19
Source File: FileUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private static void execSetPermission(File f, 
                                      FsPermission permission
                                     )  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
  } else {
    execCommand(f, Shell.getSetPermissionCommand(
                String.format("%04o", permission.toShort()), false));
  }
}
Example 20
Source File: FileUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Set permissions to the required value. Uses the java primitives instead
 * of forking if group == other.
 * @param f the file to change
 * @param permission the new permissions
 * @throws IOException
 */
public static void setPermission(File f, FsPermission permission
                                 ) throws IOException {
  FsAction user = permission.getUserAction();
  FsAction group = permission.getGroupAction();
  FsAction other = permission.getOtherAction();

  // use the native/fork if the group/other permissions are different
  // or if the native is available or on Windows
  if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
    execSetPermission(f, permission);
    return;
  }
  
  boolean rv = true;
  
  // read perms
  rv = f.setReadable(group.implies(FsAction.READ), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
    rv = f.setReadable(user.implies(FsAction.READ), true);
    checkReturnValue(rv, f, permission);
  }

  // write perms
  rv = f.setWritable(group.implies(FsAction.WRITE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
    rv = f.setWritable(user.implies(FsAction.WRITE), true);
    checkReturnValue(rv, f, permission);
  }

  // exec perms
  rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
    rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
    checkReturnValue(rv, f, permission);
  }
}