org.apache.hadoop.io.nativeio.NativeIO Java Examples

The following examples show how to use org.apache.hadoop.io.nativeio.NativeIO. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FsDatasetImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Bump a replica's generation stamp to a new one.
 * Its on-disk meta file name is renamed to be the new one too.
 * 
 * @param replicaInfo a replica
 * @param newGS new generation stamp
 * @throws IOException if rename fails
 */
private void bumpReplicaGS(ReplicaInfo replicaInfo, 
    long newGS) throws IOException { 
  long oldGS = replicaInfo.getGenerationStamp();
  File oldmeta = replicaInfo.getMetaFile();
  replicaInfo.setGenerationStamp(newGS);
  File newmeta = replicaInfo.getMetaFile();

  // rename meta file to new GS
  if (LOG.isDebugEnabled()) {
    LOG.debug("Renaming " + oldmeta + " to " + newmeta);
  }
  try {
    NativeIO.renameTo(oldmeta, newmeta);
  } catch (IOException e) {
    replicaInfo.setGenerationStamp(oldGS); // restore old GS
    throw new IOException("Block " + replicaInfo + " reopen failed. " +
                          " Unable to move meta file  " + oldmeta +
                          " to " + newmeta, e);
  }
}
 
Example #2
Source File: TestFsDatasetCache.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Run testCacheAndUncacheBlock with some failures injected into the mlock
 * call.  This tests the ability of the NameNode to resend commands.
 */
@Test(timeout=600000)
public void testCacheAndUncacheBlockWithRetries() throws Exception {
  // We don't have to save the previous cacheManipulator
  // because it will be reinstalled by the @After function.
  NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator() {
    private final Set<String> seenIdentifiers = new HashSet<String>();
    
    @Override
    public void mlock(String identifier,
        ByteBuffer mmap, long length) throws IOException {
      if (seenIdentifiers.contains(identifier)) {
        // mlock succeeds the second time.
        LOG.info("mlocking " + identifier);
        return;
      }
      seenIdentifiers.add(identifier);
      throw new IOException("injecting IOException during mlock of " +
          identifier);
    }
  });
  testCacheAndUncacheBlock();
}
 
Example #3
Source File: FsDatasetImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Bump a replica's generation stamp to a new one.
 * Its on-disk meta file name is renamed to be the new one too.
 * 
 * @param replicaInfo a replica
 * @param newGS new generation stamp
 * @throws IOException if rename fails
 */
private void bumpReplicaGS(ReplicaInfo replicaInfo, 
    long newGS) throws IOException { 
  long oldGS = replicaInfo.getGenerationStamp();
  File oldmeta = replicaInfo.getMetaFile();
  replicaInfo.setGenerationStamp(newGS);
  File newmeta = replicaInfo.getMetaFile();

  // rename meta file to new GS
  if (LOG.isDebugEnabled()) {
    LOG.debug("Renaming " + oldmeta + " to " + newmeta);
  }
  try {
    NativeIO.renameTo(oldmeta, newmeta);
  } catch (IOException e) {
    replicaInfo.setGenerationStamp(oldGS); // restore old GS
    throw new IOException("Block " + replicaInfo + " reopen failed. " +
                          " Unable to move meta file  " + oldmeta +
                          " to " + newmeta, e);
  }
}
 
Example #4
Source File: MappableBlock.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Load the block.
 *
 * mmap and mlock the block, and then verify its checksum.
 *
 * @param length         The current length of the block.
 * @param blockIn        The block input stream.  Should be positioned at the
 *                       start.  The caller must close this.
 * @param metaIn         The meta file input stream.  Should be positioned at
 *                       the start.  The caller must close this.
 * @param blockFileName  The block file name, for logging purposes.
 *
 * @return               The Mappable block.
 */
public static MappableBlock load(long length,
    FileInputStream blockIn, FileInputStream metaIn,
    String blockFileName) throws IOException {
  MappableBlock mappableBlock = null;
  MappedByteBuffer mmap = null;
  FileChannel blockChannel = null;
  try {
    blockChannel = blockIn.getChannel();
    if (blockChannel == null) {
      throw new IOException("Block InputStream has no FileChannel.");
    }
    mmap = blockChannel.map(MapMode.READ_ONLY, 0, length);
    NativeIO.POSIX.getCacheManipulator().mlock(blockFileName, mmap, length);
    verifyChecksum(length, metaIn, blockChannel, blockFileName);
    mappableBlock = new MappableBlock(mmap, length);
  } finally {
    IOUtils.closeQuietly(blockChannel);
    if (mappableBlock == null) {
      if (mmap != null) {
        NativeIO.POSIX.munmap(mmap); // unmapping also unlocks
      }
    }
  }
  return mappableBlock;
}
 
Example #5
Source File: TestEnhancedByteBufferAccess.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static HdfsConfiguration initZeroCopyTest() {
  Assume.assumeTrue(NativeIO.isAvailable());
  Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
  HdfsConfiguration conf = new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE, 3);
  conf.setLong(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS, 100);
  conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
      new File(sockDir.getDir(),
        "TestRequestMmapAccess._PORT.sock").getAbsolutePath());
  conf.setBoolean(DFSConfigKeys.
      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true);
  conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
  conf.setLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 1000);
  conf.setLong(DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 1000);
  return conf;
}
 
Example #6
Source File: TestFsDatasetCache.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Run testCacheAndUncacheBlock with some failures injected into the mlock
 * call.  This tests the ability of the NameNode to resend commands.
 */
@Test(timeout=600000)
public void testCacheAndUncacheBlockWithRetries() throws Exception {
  // We don't have to save the previous cacheManipulator
  // because it will be reinstalled by the @After function.
  NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator() {
    private final Set<String> seenIdentifiers = new HashSet<String>();
    
    @Override
    public void mlock(String identifier,
        ByteBuffer mmap, long length) throws IOException {
      if (seenIdentifiers.contains(identifier)) {
        // mlock succeeds the second time.
        LOG.info("mlocking " + identifier);
        return;
      }
      seenIdentifiers.add(identifier);
      throw new IOException("injecting IOException during mlock of " +
          identifier);
    }
  });
  testCacheAndUncacheBlock();
}
 
Example #7
Source File: RawLocalFileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up
private LocalFSFileOutputStream(Path f, boolean append,
    FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
 
Example #8
Source File: FadvisedChunkedFile.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws Exception {
  if (readaheadRequest != null) {
    readaheadRequest.cancel();
  }
  if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
    try {
      NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
          fd,
          getStartOffset(), getEndOffset() - getStartOffset(),
          NativeIO.POSIX.POSIX_FADV_DONTNEED);
    } catch (Throwable t) {
      LOG.warn("Failed to manage OS cache for " + identifier, t);
    }
  }
  super.close();
}
 
Example #9
Source File: SecureIOUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Same as openForRandomRead except that it will run even if security is off.
 * This is used by unit tests.
 */
@VisibleForTesting
protected static RandomAccessFile forceSecureOpenForRandomRead(File f,
    String mode, String expectedOwner, String expectedGroup)
    throws IOException {
  RandomAccessFile raf = new RandomAccessFile(f, mode);
  boolean success = false;
  try {
    Stat stat = NativeIO.POSIX.getFstat(raf.getFD());
    checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
        expectedGroup);
    success = true;
    return raf;
  } finally {
    if (!success) {
      raf.close();
    }
  }
}
 
Example #10
Source File: FadvisedChunkedFile.java    From tajo with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws Exception {
  if (readaheadRequest != null) {
    readaheadRequest.cancel();
  }
  if (PullServerUtil.isNativeIOPossible() && manageOsCache && endOffset() - startOffset() > 0) {
    try {
      PullServerUtil.posixFadviseIfPossible(identifier,
          fd,
          startOffset(), endOffset() - startOffset(),
          NativeIO.POSIX.POSIX_FADV_DONTNEED);
    } catch (Throwable t) {
      LOG.warn("Failed to manage OS cache for " + identifier, t);
    }
  }
  super.close();
}
 
Example #11
Source File: SecureIOUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Same as openForRead() except that it will run even if security is off.
 * This is used by unit tests.
 */
@VisibleForTesting
protected static FileInputStream forceSecureOpenForRead(File f, String expectedOwner,
    String expectedGroup) throws IOException {

  FileInputStream fis = new FileInputStream(f);
  boolean success = false;
  try {
    Stat stat = NativeIO.POSIX.getFstat(fis.getFD());
    checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
        expectedGroup);
    success = true;
    return fis;
  } finally {
    if (!success) {
      fis.close();
    }
  }
}
 
Example #12
Source File: TestEnhancedByteBufferAccess.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static HdfsConfiguration initZeroCopyTest() {
  Assume.assumeTrue(NativeIO.isAvailable());
  Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
  HdfsConfiguration conf = new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE, 3);
  conf.setLong(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS, 100);
  conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
      new File(sockDir.getDir(),
        "TestRequestMmapAccess._PORT.sock").getAbsolutePath());
  conf.setBoolean(DFSConfigKeys.
      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true);
  conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
  conf.setLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 1000);
  conf.setLong(DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 1000);
  return conf;
}
 
Example #13
Source File: SecureIOUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Same as openFSDataInputStream except that it will run even if security is
 * off. This is used by unit tests.
 */
@VisibleForTesting
protected static FSDataInputStream forceSecureOpenFSDataInputStream(
    File file,
    String expectedOwner, String expectedGroup) throws IOException {
  final FSDataInputStream in =
      rawFilesystem.open(new Path(file.getAbsolutePath()));
  boolean success = false;
  try {
    Stat stat = NativeIO.POSIX.getFstat(in.getFileDescriptor());
    checkStat(file, stat.getOwner(), stat.getGroup(), expectedOwner,
        expectedGroup);
    success = true;
    return in;
  } finally {
    if (!success) {
      in.close();
    }
  }
}
 
Example #14
Source File: SecureIOUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Same as openFSDataInputStream except that it will run even if security is
 * off. This is used by unit tests.
 */
@VisibleForTesting
protected static FSDataInputStream forceSecureOpenFSDataInputStream(
    File file,
    String expectedOwner, String expectedGroup) throws IOException {
  final FSDataInputStream in =
      rawFilesystem.open(new Path(file.getAbsolutePath()));
  boolean success = false;
  try {
    Stat stat = NativeIO.POSIX.getFstat(in.getFileDescriptor());
    checkStat(file, stat.getOwner(), stat.getGroup(), expectedOwner,
        expectedGroup);
    success = true;
    return in;
  } finally {
    if (!success) {
      in.close();
    }
  }
}
 
Example #15
Source File: RawLocalFileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
    throws IOException {
  if (permission == null) {
    return p2f.mkdir();
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      try {
        NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
        return true;
      } catch (IOException e) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format(
              "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
              p2f, permission.toShort()), e);
        }
        return false;
      }
    } else {
      boolean b = p2f.mkdir();
      if (b) {
        setPermission(p, permission);
      }
      return b;
    }
  }
}
 
Example #16
Source File: ReadaheadPool.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public void run() {
  if (canceled) return;
  // There's a very narrow race here that the file will close right at
  // this instant. But if that happens, we'll likely receive an EBADF
  // error below, and see that it's canceled, ignoring the error.
  // It's also possible that we'll end up requesting readahead on some
  // other FD, which may be wasted work, but won't cause a problem.
  try {
    NativeIO.posixFadviseIfPossible(fd, off, len,
        NativeIO.POSIX_FADV_WILLNEED);
  } catch (IOException ioe) {
    if (canceled) {
      // no big deal - the reader canceled the request and closed
      // the file.
      return;
    }
    LOG.warn("Failed readahead on " + identifier,
        ioe);
  }
}
 
Example #17
Source File: ReadaheadPool.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  if (canceled) return;
  // There's a very narrow race here that the file will close right at
  // this instant. But if that happens, we'll likely receive an EBADF
  // error below, and see that it's canceled, ignoring the error.
  // It's also possible that we'll end up requesting readahead on some
  // other FD, which may be wasted work, but won't cause a problem.
  try {
    NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
        fd, off, len, NativeIO.POSIX.POSIX_FADV_WILLNEED);
  } catch (IOException ioe) {
    if (canceled) {
      // no big deal - the reader canceled the request and closed
      // the file.
      return;
    }
    LOG.warn("Failed readahead on " + identifier,
        ioe);
  }
}
 
Example #18
Source File: ReadaheadPool.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  if (canceled) return;
  // There's a very narrow race here that the file will close right at
  // this instant. But if that happens, we'll likely receive an EBADF
  // error below, and see that it's canceled, ignoring the error.
  // It's also possible that we'll end up requesting readahead on some
  // other FD, which may be wasted work, but won't cause a problem.
  try {
    NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
        fd, off, len, NativeIO.POSIX.POSIX_FADV_WILLNEED);
  } catch (IOException ioe) {
    if (canceled) {
      // no big deal - the reader canceled the request and closed
      // the file.
      return;
    }
    LOG.warn("Failed readahead on " + identifier,
        ioe);
  }
}
 
Example #19
Source File: FadvisedChunkedFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws Exception {
  if (readaheadRequest != null) {
    readaheadRequest.cancel();
  }
  if (manageOsCache && getEndOffset() - getStartOffset() > 0) {
    try {
      NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
          fd,
          getStartOffset(), getEndOffset() - getStartOffset(),
          NativeIO.POSIX.POSIX_FADV_DONTNEED);
    } catch (Throwable t) {
      LOG.warn("Failed to manage OS cache for " + identifier, t);
    }
  }
  super.close();
}
 
Example #20
Source File: RawLocalFileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private LocalFSFileOutputStream(Path f, boolean append,
    FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
 
Example #21
Source File: FileUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Platform independent implementation for {@link File#canRead()}
 * @param f input file
 * @return On Unix, same as {@link File#canRead()}
 *         On Windows, true if process has read access on the path
 */
public static boolean canRead(File f) {
  if (Shell.WINDOWS) {
    try {
      return NativeIO.Windows.access(f.getCanonicalPath(),
          NativeIO.Windows.AccessRight.ACCESS_READ);
    } catch (IOException e) {
      return false;
    }
  } else {
    return f.canRead();
  }
}
 
Example #22
Source File: DFSTestUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Blocks until cache usage hits the expected new value.
 */
public static long verifyExpectedCacheUsage(final long expectedCacheUsed,
    final long expectedBlocks, final FsDatasetSpi<?> fsd) throws Exception {
  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    private int tries = 0;
    
    @Override
    public Boolean get() {
      long curCacheUsed = fsd.getCacheUsed();
      long curBlocks = fsd.getNumBlocksCached();
      if ((curCacheUsed != expectedCacheUsed) ||
          (curBlocks != expectedBlocks)) {
        if (tries++ > 10) {
          LOG.info("verifyExpectedCacheUsage: have " +
              curCacheUsed + "/" + expectedCacheUsed + " bytes cached; " +
              curBlocks + "/" + expectedBlocks + " blocks cached. " +
              "memlock limit = " +
              NativeIO.POSIX.getCacheManipulator().getMemlockLimit() +
              ".  Waiting...");
        }
        return false;
      }
      LOG.info("verifyExpectedCacheUsage: got " +
          curCacheUsed + "/" + expectedCacheUsed + " bytes cached; " +
          curBlocks + "/" + expectedBlocks + " blocks cached. " +
          "memlock limit = " +
          NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
      return true;
    }
  }, 100, 60000);
  return expectedCacheUsed;
}
 
Example #23
Source File: TestCacheDirectives.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  conf = createCachingConf();
  cluster =
      new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
  cluster.waitActive();
  dfs = cluster.getFileSystem();
  proto = cluster.getNameNodeRpc();
  namenode = cluster.getNameNode();
  prevCacheManipulator = NativeIO.POSIX.getCacheManipulator();
  NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
  BlockReaderTestUtil.enableHdfsCachingTracing();
}
 
Example #24
Source File: FileUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Set permissions to the required value. Uses the java primitives instead
 * of forking if group == other.
 * @param f the file to change
 * @param permission the new permissions
 * @throws IOException
 */
public static void setPermission(File f, FsPermission permission
                                 ) throws IOException {
  FsAction user = permission.getUserAction();
  FsAction group = permission.getGroupAction();
  FsAction other = permission.getOtherAction();

  // use the native/fork if the group/other permissions are different
  // or if the native is available or on Windows
  if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
    execSetPermission(f, permission);
    return;
  }
  
  boolean rv = true;
  
  // read perms
  rv = f.setReadable(group.implies(FsAction.READ), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
    rv = f.setReadable(user.implies(FsAction.READ), true);
    checkReturnValue(rv, f, permission);
  }

  // write perms
  rv = f.setWritable(group.implies(FsAction.WRITE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
    rv = f.setWritable(user.implies(FsAction.WRITE), true);
    checkReturnValue(rv, f, permission);
  }

  // exec perms
  rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
  checkReturnValue(rv, f, permission);
  if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
    rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
    checkReturnValue(rv, f, permission);
  }
}
 
Example #25
Source File: ShortCircuitShm.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Create the ShortCircuitShm.
 * 
 * @param shmId       The ID to use.
 * @param stream      The stream that we're going to use to create this 
 *                    shared memory segment.
 *                    
 *                    Although this is a FileInputStream, we are going to
 *                    assume that the underlying file descriptor is writable
 *                    as well as readable. It would be more appropriate to use
 *                    a RandomAccessFile here, but that class does not have
 *                    any public accessor which returns a FileDescriptor,
 *                    unlike FileInputStream.
 */
public ShortCircuitShm(ShmId shmId, FileInputStream stream)
      throws IOException {
  if (!NativeIO.isAvailable()) {
    throw new UnsupportedOperationException("NativeIO is not available.");
  }
  if (Shell.WINDOWS) {
    throw new UnsupportedOperationException(
        "DfsClientShm is not yet implemented for Windows.");
  }
  if (unsafe == null) {
    throw new UnsupportedOperationException(
        "can't use DfsClientShm because we failed to " +
        "load misc.Unsafe.");
  }
  this.shmId = shmId;
  this.mmappedLength = getUsableLength(stream);
  this.baseAddress = POSIX.mmap(stream.getFD(), 
      POSIX.MMAP_PROT_READ | POSIX.MMAP_PROT_WRITE, true, mmappedLength);
  this.slots = new Slot[mmappedLength / BYTES_PER_SLOT];
  this.allocatedSlots = new BitSet(slots.length);
  if (LOG.isTraceEnabled()) {
    LOG.trace("creating " + this.getClass().getSimpleName() +
        "(shmId=" + shmId +
        ", mmappedLength=" + mmappedLength +
        ", baseAddress=" + String.format("%x", baseAddress) +
        ", slots.length=" + slots.length + ")");
  }
}
 
Example #26
Source File: RawLocalFileSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Use the command chmod to set permission.
 */
@Override
public void setPermission(Path p, FsPermission permission)
  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                   permission.toShort());
  } else {
    String perm = String.format("%04o", permission.toShort());
    Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
      FileUtil.makeShellPath(pathToFile(p), true)));
  }
}
 
Example #27
Source File: FileUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Platform independent implementation for {@link File#canWrite()}
 * @param f input file
 * @return On Unix, same as {@link File#canWrite()}
 *         On Windows, true if process has write access on the path
 */
public static boolean canWrite(File f) {
  if (Shell.WINDOWS) {
    try {
      return NativeIO.Windows.access(f.getCanonicalPath(),
          NativeIO.Windows.AccessRight.ACCESS_WRITE);
    } catch (IOException e) {
      return false;
    }
  } else {
    return f.canWrite();
  }
}
 
Example #28
Source File: RawLocalFileSystem.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private LocalFSFileOutputStream(Path f, boolean append,
                                FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (!append && permission == null) {
    permission = FsPermission.getFileDefault();
  }
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    permission = permission.applyUMask(FsPermission.getUMask(getConf()));
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
 
Example #29
Source File: FileUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Platform independent implementation for {@link File#canExecute()}
 * @param f input file
 * @return On Unix, same as {@link File#canExecute()}
 *         On Windows, true if process has execute access on the path
 */
public static boolean canExecute(File f) {
  if (Shell.WINDOWS) {
    try {
      return NativeIO.Windows.access(f.getCanonicalPath(),
          NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
    } catch (IOException e) {
      return false;
    }
  } else {
    return f.canExecute();
  }
}
 
Example #30
Source File: FileUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void execSetPermission(File f, 
                                      FsPermission permission
                                     )  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
  } else {
    execCommand(f, Shell.getSetPermissionCommand(
                String.format("%04o", permission.toShort()), false));
  }
}