org.apache.hadoop.fs.Options Java Examples

The following examples show how to use org.apache.hadoop.fs.Options. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FileSystemWAL.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
private DataOutputStream getOutputStream(FileSystemWALPointer pointer) throws IOException
{
  Preconditions.checkArgument(outputStream == null, "output stream is not null");

  if (pointer.offset > 0 && (fileSystemWAL.fileContext.getDefaultFileSystem() instanceof LocalFs ||
      fileSystemWAL.fileContext.getDefaultFileSystem() instanceof RawLocalFs)) {
    //On local file system the stream is always closed and never flushed so we open it again in append mode if the
    //offset > 0. This block is entered only when appending to wal while writing on local fs.
    return fileSystemWAL.fileContext.create(new Path(fileSystemWAL.tempPartFiles.get(pointer.partNum)),
        EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND), Options.CreateOpts.CreateParent.createParent());
  }

  String partFile = fileSystemWAL.getPartFilePath(pointer.partNum);
  String tmpFilePath = createTmpFilePath(partFile);
  fileSystemWAL.tempPartFiles.put(pointer.partNum, tmpFilePath);

  Preconditions.checkArgument(pointer.offset == 0, "offset > 0");
  LOG.debug("open {} => {}", pointer.partNum, tmpFilePath);
  outputStream = fileSystemWAL.fileContext.create(new Path(tmpFilePath),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());
  return outputStream;
}
 
Example #2
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testOpen() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    int cnt = 2 * 1024;

    try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()))) {

        for (long i = 0; i < cnt; i++)
            out.writeLong(i);
    }

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    try (FSDataInputStream in = fs.open(file, 1024)) {

        for (long i = 0; i < cnt; i++)
            assertEquals(i, in.readLong());
    }
}
 
Example #3
Source File: NameNodeRpcServer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 
Example #4
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    fs.setOwner(file, "fUser", "fGroup");
    fs.setOwner(tmpDir, "dUser", "dGroup");

    assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
    assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());

    assertEquals("fUser", fs.getFileStatus(file).getOwner());
    assertEquals("fGroup", fs.getFileStatus(file).getGroup());
}
 
Example #5
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testOpenIfPathIsAlreadyOpened() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataInputStream is1 = fs.open(file);
    FSDataInputStream is2 = fs.open(file);

    is1.close();
    is2.close();
}
 
Example #6
Source File: DFSClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 
Example #7
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testAppendIfPathPointsToDirectory() throws Exception {
    final Path fsHome = new Path(primaryFsUri);
    final Path dir = new Path(fsHome, "/tmp");
    Path file = new Path(dir, "my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);
}
 
Example #8
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(file, EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);

    appendOs.close();
}
 
Example #9
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testRenameCheckParametersDstPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault())).close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.rename(file, null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: f");
}
 
Example #10
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwner() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    fs.setOwner(file, "aUser", "aGroup");

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 
Example #11
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setOwner(file, "aUser", "aGroup");

    os.close();

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 
Example #12
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, "aUser", null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
}
 
Example #13
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersUserIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, null, "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: username");
}
 
Example #14
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FsPermission perm = new FsPermission((short)123);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setPermission(file, perm);

    os.close();

    assertEquals(perm, fs.getFileStatus(file).getPermission());
}
 
Example #15
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testRenameDirectory() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/tmp/");
    Path newDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.rename(dir, newDir);

    assertPathDoesNotExist(fs, dir);
    assertPathExists(fs, newDir);
}
 
Example #16
Source File: IOUtilsTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException
  {
    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
    File file = new File("target/IOUtilsTest/testCopyPartial/input");
    createDataFile(file, dataSize);

    FileContext fileContext = FileContext.getFileContext();
    DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));

    Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
    DataOutputStream outputStream = fileContext.create(output, EnumSet
        .of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());

    if (offset == 0) {
      IOUtils.copyPartial(inputStream, size, outputStream);
    } else {
      IOUtils.copyPartial(inputStream, offset, size, outputStream);
    }

    outputStream.close();

    Assert.assertTrue("output exists", fileContext.util().exists(output));
    Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
//    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
  }
 
Example #17
Source File: NameNodeRpcServer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 
Example #18
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testCreateBase() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
    Path file = new Path(dir, "someFile");

    assertPathDoesNotExist(fs, file);

    FsPermission fsPerm = new FsPermission((short)644);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(fsPerm));

    // Try to write something in file.
    os.write("abc".getBytes());

    os.close();

    // Check file status.
    FileStatus fileStatus = fs.getFileStatus(file);

    assertFalse(fileStatus.isDirectory());
    assertEquals(file, fileStatus.getPath());
    assertEquals(fsPerm, fileStatus.getPermission());
}
 
Example #19
Source File: DFSClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 
Example #20
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testDeleteFailsIfNonRecursive() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.delete(someDir2, false);

            return null;
        }
    }, PathIsNotEmptyDirectoryException.class, null);

    assertPathExists(fs, someDir2);
    assertPathExists(fs, someDir3);
}
 
Example #21
Source File: HadoopUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * A wrapper around {@link FileSystem#rename(Path, Path)} which throws {@link IOException} if
 * {@link FileSystem#rename(Path, Path)} returns False.
 */
public static void renamePath(FileSystem fs, Path oldName, Path newName, boolean overwrite) throws IOException {
  //In default implementation of rename with rewrite option in FileSystem, if the parent dir of dst does not exist, it will throw exception,
  //Which will fail some of our job unintentionally. So we only call that method when fs is an instance of DistributedFileSystem to avoid inconsistency problem
  if(fs instanceof DistributedFileSystem) {
    Options.Rename renameOptions = (overwrite) ? Options.Rename.OVERWRITE : Options.Rename.NONE;
    ((DistributedFileSystem) fs).rename(oldName, newName, renameOptions);
  } else {
    if (!fs.exists(oldName)) {
      throw new FileNotFoundException(String.format("Failed to rename %s to %s: src not found", oldName, newName));
    }
    if (fs.exists(newName)) {
      if (overwrite) {
        HadoopUtils.moveToTrash(fs, newName);
      } else {
        throw new FileAlreadyExistsException(String.format("Failed to rename %s to %s: dst already exists", oldName, newName));
      }
    }
    if (!fs.rename(oldName, newName)) {
      throw new IOException(String.format("Failed to rename %s to %s", oldName, newName));
    }
  }
}
 
Example #22
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testDeleteRecursivelyFromRoot() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path root = new Path(fsHome, "/");

    assertFalse(fs.delete(root, true));

    assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));

    assertPathDoesNotExist(fs, someDir3);
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
    assertPathExists(fs, root);
}
 
Example #23
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckDefaultPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.setPermission(file, null);

    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
}
 
Example #24
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    FsPermission perm = new FsPermission((short)123);

    fs.setPermission(tmpDir, perm);

    assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
}
 
Example #25
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@SuppressWarnings("OctalInteger")
@Test
public void testSetPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    for (short i = 0; i <= 0777; i += 7) {
        FsPermission perm = new FsPermission(i);

        fs.setPermission(file, perm);

        assertEquals(perm, fs.getFileStatus(file).getPermission());
    }
}
 
Example #26
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(null, "aUser", "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: p");
}
 
Example #27
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 5 votes vote down vote up
/** @throws Exception If failed. */
@Test
public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path srcFile = new Path(fsHome, "srcFile");
    Path dstFile = new Path(fsHome, "dstFile");

    FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    int cnt = 1024;

    for (int i = 0; i < cnt; i++)
        os.writeInt(i);

    os.close();

    FSDataInputStream is = fs.open(srcFile);

    for (int i = 0; i < cnt; i++) {
        if (i == 100)
            // Rename file during the read process.
            fs.rename(srcFile, dstFile);

        assertEquals(i, is.readInt());
    }

    assertPathDoesNotExist(fs, srcFile);
    assertPathExists(fs, dstFile);

    os.close();
    is.close();
}
 
Example #28
Source File: HadoopIgfs20FileSystemAbstractSelfTest.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * @throws Exception If failed.
 */
@Test
public void testRenameDirectoryIfDstPathExists() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path srcDir = new Path(fsHome, "/tmp/");
    Path dstDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(srcDir, "file1"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    os = fs.create(new Path(dstDir, "file2"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    try {
        fs.rename(srcDir, dstDir);

        fail("FileAlreadyExistsException expected.");
    }
    catch (FileAlreadyExistsException ignore) {
        // No-op.
    }

    // Check all the files stay unchanged:
    assertPathExists(fs, dstDir);
    assertPathExists(fs, new Path(dstDir, "file2"));

    assertPathExists(fs, srcDir);
    assertPathExists(fs, new Path(srcDir, "file1"));
}
 
Example #29
Source File: FSStorageAgent.java    From Bats with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
  String operatorIdStr = String.valueOf(operatorId);
  Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
  String window = Long.toHexString(windowId);
  boolean stateSaved = false;
  FSDataOutputStream stream = null;
  try {
    stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      Options.CreateOpts.CreateParent.createParent());
    store(stream, object);
    stateSaved = true;
  } catch (Throwable t) {
    logger.debug("while saving {} {}", operatorId, window, t);
    stateSaved = false;
    throw Throwables.propagate(t);
  } finally {
    try {
      if (stream != null) {
        stream.close();
      }
    } catch (IOException ie) {
      stateSaved = false;
      throw new RuntimeException(ie);
    } finally {
      if (stateSaved) {
        logger.debug("Saving {}: {}", operatorId, window);
        fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
            Options.Rename.OVERWRITE);
      }
    }
  }
}
 
Example #30
Source File: FileSystemWAL.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
protected void finalize(int partNum) throws IOException
{
  String tmpToFinalize = fileSystemWAL.tempPartFiles.remove(partNum);
  Path tmpPath = new Path(tmpToFinalize);
  if (fileSystemWAL.fileContext.util().exists(tmpPath)) {
    LOG.debug("finalize {} of part {}", tmpPath, partNum);
    fileSystemWAL.fileContext.rename(tmpPath, new Path(fileSystemWAL.getPartFilePath(partNum)),
        Options.Rename.OVERWRITE);
    latestFinalizedPart = partNum;
  }
}