Java Code Examples for org.apache.hadoop.fs.Options

The following examples show how to use org.apache.hadoop.fs.Options. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 
Example 2
Source Project: hadoop   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 
Example 3
Source Project: big-c   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 
Example 4
Source Project: big-c   Source File: NameNodeRpcServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 
Example 5
Source Project: attic-apex-malhar   Source File: FileSystemWAL.java    License: Apache License 2.0 6 votes vote down vote up
private DataOutputStream getOutputStream(FileSystemWALPointer pointer) throws IOException
{
  Preconditions.checkArgument(outputStream == null, "output stream is not null");

  if (pointer.offset > 0 && (fileSystemWAL.fileContext.getDefaultFileSystem() instanceof LocalFs ||
      fileSystemWAL.fileContext.getDefaultFileSystem() instanceof RawLocalFs)) {
    //On local file system the stream is always closed and never flushed so we open it again in append mode if the
    //offset > 0. This block is entered only when appending to wal while writing on local fs.
    return fileSystemWAL.fileContext.create(new Path(fileSystemWAL.tempPartFiles.get(pointer.partNum)),
        EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND), Options.CreateOpts.CreateParent.createParent());
  }

  String partFile = fileSystemWAL.getPartFilePath(pointer.partNum);
  String tmpFilePath = createTmpFilePath(partFile);
  fileSystemWAL.tempPartFiles.put(pointer.partNum, tmpFilePath);

  Preconditions.checkArgument(pointer.offset == 0, "offset > 0");
  LOG.debug("open {} => {}", pointer.partNum, tmpFilePath);
  outputStream = fileSystemWAL.fileContext.create(new Path(tmpFilePath),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());
  return outputStream;
}
 
Example 6
Source Project: attic-apex-malhar   Source File: IOUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException
  {
    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
    File file = new File("target/IOUtilsTest/testCopyPartial/input");
    createDataFile(file, dataSize);

    FileContext fileContext = FileContext.getFileContext();
    DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));

    Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
    DataOutputStream outputStream = fileContext.create(output, EnumSet
        .of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());

    if (offset == 0) {
      IOUtils.copyPartial(inputStream, size, outputStream);
    } else {
      IOUtils.copyPartial(inputStream, offset, size, outputStream);
    }

    outputStream.close();

    Assert.assertTrue("output exists", fileContext.util().exists(output));
    Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
//    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
  }
 
Example 7
/** @throws Exception If failed. */
@Test
public void testCreateBase() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
    Path file = new Path(dir, "someFile");

    assertPathDoesNotExist(fs, file);

    FsPermission fsPerm = new FsPermission((short)644);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(fsPerm));

    // Try to write something in file.
    os.write("abc".getBytes());

    os.close();

    // Check file status.
    FileStatus fileStatus = fs.getFileStatus(file);

    assertFalse(fileStatus.isDirectory());
    assertEquals(file, fileStatus.getPath());
    assertEquals(fsPerm, fileStatus.getPermission());
}
 
Example 8
/** @throws Exception If failed. */
@Test
public void testDeleteFailsIfNonRecursive() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.delete(someDir2, false);

            return null;
        }
    }, PathIsNotEmptyDirectoryException.class, null);

    assertPathExists(fs, someDir2);
    assertPathExists(fs, someDir3);
}
 
Example 9
Source Project: incubator-gobblin   Source File: HadoopUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * A wrapper around {@link FileSystem#rename(Path, Path)} which throws {@link IOException} if
 * {@link FileSystem#rename(Path, Path)} returns False.
 */
public static void renamePath(FileSystem fs, Path oldName, Path newName, boolean overwrite) throws IOException {
  //In default implementation of rename with rewrite option in FileSystem, if the parent dir of dst does not exist, it will throw exception,
  //Which will fail some of our job unintentionally. So we only call that method when fs is an instance of DistributedFileSystem to avoid inconsistency problem
  if(fs instanceof DistributedFileSystem) {
    Options.Rename renameOptions = (overwrite) ? Options.Rename.OVERWRITE : Options.Rename.NONE;
    ((DistributedFileSystem) fs).rename(oldName, newName, renameOptions);
  } else {
    if (!fs.exists(oldName)) {
      throw new FileNotFoundException(String.format("Failed to rename %s to %s: src not found", oldName, newName));
    }
    if (fs.exists(newName)) {
      if (overwrite) {
        HadoopUtils.moveToTrash(fs, newName);
      } else {
        throw new FileAlreadyExistsException(String.format("Failed to rename %s to %s: dst already exists", oldName, newName));
      }
    }
    if (!fs.rename(oldName, newName)) {
      throw new IOException(String.format("Failed to rename %s to %s", oldName, newName));
    }
  }
}
 
Example 10
/** @throws Exception If failed. */
@Test
public void testDeleteRecursivelyFromRoot() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path root = new Path(fsHome, "/");

    assertFalse(fs.delete(root, true));

    assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));

    assertPathDoesNotExist(fs, someDir3);
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
    assertPathExists(fs, root);
}
 
Example 11
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckDefaultPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.setPermission(file, null);

    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
}
 
Example 12
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    FsPermission perm = new FsPermission((short)123);

    fs.setPermission(tmpDir, perm);

    assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
}
 
Example 13
/** @throws Exception If failed. */
@SuppressWarnings("OctalInteger")
@Test
public void testSetPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    for (short i = 0; i <= 0777; i += 7) {
        FsPermission perm = new FsPermission(i);

        fs.setPermission(file, perm);

        assertEquals(perm, fs.getFileStatus(file).getPermission());
    }
}
 
Example 14
/** @throws Exception If failed. */
@Test
public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FsPermission perm = new FsPermission((short)123);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setPermission(file, perm);

    os.close();

    assertEquals(perm, fs.getFileStatus(file).getPermission());
}
 
Example 15
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(null, "aUser", "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: p");
}
 
Example 16
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersUserIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, null, "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: username");
}
 
Example 17
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, "aUser", null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
}
 
Example 18
/** @throws Exception If failed. */
@Test
public void testSetOwner() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    fs.setOwner(file, "aUser", "aGroup");

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 
Example 19
/** @throws Exception If failed. */
@Test
public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setOwner(file, "aUser", "aGroup");

    os.close();

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 
Example 20
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    fs.setOwner(file, "fUser", "fGroup");
    fs.setOwner(tmpDir, "dUser", "dGroup");

    assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
    assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());

    assertEquals("fUser", fs.getFileStatus(file).getOwner());
    assertEquals("fGroup", fs.getFileStatus(file).getGroup());
}
 
Example 21
/** @throws Exception If failed. */
@Test
public void testOpenIfPathIsAlreadyOpened() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataInputStream is1 = fs.open(file);
    FSDataInputStream is2 = fs.open(file);

    is1.close();
    is2.close();
}
 
Example 22
/** @throws Exception If failed. */
@Test
public void testOpen() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    int cnt = 2 * 1024;

    try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()))) {

        for (long i = 0; i < cnt; i++)
            out.writeLong(i);
    }

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    try (FSDataInputStream in = fs.open(file, 1024)) {

        for (long i = 0; i < cnt; i++)
            assertEquals(i, in.readLong());
    }
}
 
Example 23
/** @throws Exception If failed. */
@Test
public void testAppendIfPathPointsToDirectory() throws Exception {
    final Path fsHome = new Path(primaryFsUri);
    final Path dir = new Path(fsHome, "/tmp");
    Path file = new Path(dir, "my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);
}
 
Example 24
/** @throws Exception If failed. */
@Test
public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(file, EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);

    appendOs.close();
}
 
Example 25
/** @throws Exception If failed. */
@Test
public void testRenameCheckParametersDstPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault())).close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.rename(file, null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: f");
}
 
Example 26
/** @throws Exception If failed. */
@Test
public void testRenameDirectory() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/tmp/");
    Path newDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.rename(dir, newDir);

    assertPathDoesNotExist(fs, dir);
    assertPathExists(fs, newDir);
}
 
Example 27
Source Project: Bats   Source File: FSStorageAgent.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
  String operatorIdStr = String.valueOf(operatorId);
  Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
  String window = Long.toHexString(windowId);
  boolean stateSaved = false;
  FSDataOutputStream stream = null;
  try {
    stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      Options.CreateOpts.CreateParent.createParent());
    store(stream, object);
    stateSaved = true;
  } catch (Throwable t) {
    logger.debug("while saving {} {}", operatorId, window, t);
    stateSaved = false;
    throw Throwables.propagate(t);
  } finally {
    try {
      if (stream != null) {
        stream.close();
      }
    } catch (IOException ie) {
      stateSaved = false;
      throw new RuntimeException(ie);
    } finally {
      if (stateSaved) {
        logger.debug("Saving {}: {}", operatorId, window);
        fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
            Options.Rename.OVERWRITE);
      }
    }
  }
}
 
Example 28
Source Project: garmadon   Source File: ProtoParquetWriterWithOffset.java    License: Apache License 2.0 5 votes vote down vote up
protected void moveToFinalPath(Path tempPath, Path finalPath) throws IOException {
    if (fs instanceof DistributedFileSystem) {
        ((DistributedFileSystem) fs).rename(tempPath, finalPath, Options.Rename.OVERWRITE);
    } else {
        if (!fs.rename(tempPath, finalPath)) {
            throw new IOException(String.format("Failed to commit %s (from %s)",
                finalPath.toUri(), tempPath));
        }
    }

    LOGGER.info("Committed {} to {}", tempPath, finalPath.toUri());
}
 
Example 29
@Test
public void testCreateInternal_shouldCreateParent() throws Exception {
  Configuration config = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
  GoogleHadoopFS ghfs = new GoogleHadoopFS(initUri, config);

  Path filePath =
      new Path(initUri.resolve("/testCreateInternal_shouldCreateParent/dir/file").toString());

  try (FSDataOutputStream stream =
      ghfs.createInternal(
          filePath,
          EnumSet.of(CreateFlag.CREATE),
          /* absolutePermission= */ null,
          /* bufferSize= */ 128,
          /* replication= */ (short) 1,
          /* blockSize= */ 32,
          () -> {},
          new Options.ChecksumOpt(),
          /* createParent= */ true)) {
    stream.write(1);

    assertThat(stream.size()).isEqualTo(1);
  }

  FileStatus parentStatus = ghfs.getFileStatus(filePath.getParent());
  assertThat(parentStatus.getModificationTime()).isGreaterThan(0L);
}
 
Example 30
Source Project: hadoop   Source File: AggregatedLogFormat.java    License: Apache License 2.0 5 votes vote down vote up
public LogWriter(final Configuration conf, final Path remoteAppLogFile,
    UserGroupInformation userUgi) throws IOException {
  try {
    this.fsDataOStream =
        userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() {
          @Override
          public FSDataOutputStream run() throws Exception {
            fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
            fc.setUMask(APP_LOG_FILE_UMASK);
            return fc.create(
                remoteAppLogFile,
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                new Options.CreateOpts[] {});
          }
        });
  } catch (InterruptedException e) {
    throw new IOException(e);
  }

  // Keys are not sorted: null arg
  // 256KB minBlockSize : Expected log size for each container too
  this.writer =
      new TFile.Writer(this.fsDataOStream, 256 * 1024, conf.get(
          YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE,
          YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf);
  //Write the version string
  writeVersion();
}