Java Code Examples for org.apache.ratis.util.FileUtils

The following examples show how to use org.apache.ratis.util.FileUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop-ozone   Source File: OzoneManagerRatisUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify transaction info with provided lastAppliedIndex.
 *
 * If transaction info transaction Index is less than or equal to
 * lastAppliedIndex, return false, else return true.
 * @param omTransactionInfo
 * @param lastAppliedIndex
 * @param leaderId
 * @param newDBlocation
 * @return boolean
 */
public static boolean verifyTransactionInfo(
    OMTransactionInfo omTransactionInfo,
    long lastAppliedIndex,
    String leaderId, Path newDBlocation) {
  if (omTransactionInfo.getTransactionIndex() <= lastAppliedIndex) {
    OzoneManager.LOG.error("Failed to install checkpoint from OM leader: {}" +
            ". The last applied index: {} is greater than or equal to the " +
            "checkpoint's applied index: {}. Deleting the downloaded " +
            "checkpoint {}", leaderId, lastAppliedIndex,
        omTransactionInfo.getTransactionIndex(), newDBlocation);
    try {
      FileUtils.deleteFully(newDBlocation);
    } catch (IOException e) {
      OzoneManager.LOG.error("Failed to fully delete the downloaded DB " +
          "checkpoint {} from OM leader {}.", newDBlocation, leaderId, e);
    }
    return false;
  }

  return true;
}
 
Example 2
Source Project: incubator-ratis   Source File: TestLogSegment.java    License: Apache License 2.0 5 votes vote down vote up
File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten)
    throws IOException {
  if (!isOpen) {
    Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written.");
  }
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  final File file = isOpen ?
      storage.getStorageDir().getOpenLogFile(startIndex) :
      storage.getStorageDir().getClosedLogFile(startIndex, startIndex + numEntries - 1);

  final LogEntryProto[] entries = new LogEntryProto[numEntries];
  try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false,
      segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) {
    for (int i = 0; i < entries.length; i++) {
      SimpleOperation op = new SimpleOperation("m" + i);
      entries[i] = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + startIndex);
      out.write(entries[i]);
    }
  }

  if (isLastEntryPartiallyWritten) {
    final int entrySize = size(entries[entries.length - 1]);
    final int truncatedEntrySize = ThreadLocalRandom.current().nextInt(entrySize - 1) + 1;
    // 0 < truncatedEntrySize < entrySize
    final long fileLength = file.length();
    final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize);
    LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})",
        entrySize, truncatedEntrySize, fileLength, truncatedFileLength);
    FileUtils.truncateFile(file, truncatedFileLength);
  }

  storage.close();
  return file;
}
 
Example 3
Source Project: incubator-ratis   Source File: TestRaftStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * check if RaftStorage deletes tmp metafile when startup
 */
@Test
public void testCleanMetaTmpFile() throws Exception {
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  Assert.assertEquals(StorageState.NORMAL, storage.getState());
  storage.close();

  RaftStorageDirectory sd = new RaftStorageDirectory(storageDir);
  File metaFile = sd.getMetaFile();
  FileUtils.move(metaFile, sd.getMetaTmpFile());

  Assert.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false));

  try {
    new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.fail("should throw IOException since storage dir is not formatted");
  } catch (IOException e) {
    Assert.assertTrue(
        e.getMessage().contains(StorageState.NOT_FORMATTED.name()));
  }

  // let the storage dir contain both raft-meta and raft-meta.tmp
  new RaftStorage(storageDir, StartupOption.FORMAT).close();
  Assert.assertTrue(sd.getMetaFile().exists());
  Assert.assertTrue(sd.getMetaTmpFile().createNewFile());
  Assert.assertTrue(sd.getMetaTmpFile().exists());
  try {
    storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.assertEquals(StorageState.NORMAL, storage.getState());
    Assert.assertFalse(sd.getMetaTmpFile().exists());
    Assert.assertTrue(sd.getMetaFile().exists());
  } finally {
    storage.close();
  }
}
 
Example 4
Source Project: incubator-ratis   Source File: ServerRestartTests.java    License: Apache License 2.0 5 votes vote down vote up
static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception {
  // truncate log
  FileUtils.truncateFile(openLogFile, openLogFile.length() - 1);
  final RaftServerImpl server = cluster.restartServer(id, false);
  // the last index should be one less than before
  Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex());
  server.getProxy().close();
}
 
Example 5
Source Project: incubator-ratis   Source File: FileStoreStateMachine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage)
    throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  FileUtils.createDirectories(files.getRoot());
}
 
Example 6
Source Project: incubator-ratis   Source File: FileStore.java    License: Apache License 2.0 5 votes vote down vote up
CompletableFuture<Path> delete(long index, String relative) {
  final Supplier<String> name = () -> "delete(" + relative + ") @" + getId() + ":" + index;
  final CheckedSupplier<Path, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.remove(relative);
    FileUtils.delete(resolve(info.getRelativePath()));
    return info.getRelativePath();
  }, name);
  return submit(task, deleter);
}
 
Example 7
Source Project: incubator-ratis   Source File: SimpleStateMachineStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) throws IOException {
  if (snapshotRetentionPolicy != null && snapshotRetentionPolicy.getNumSnapshotsRetained() > 0) {

    List<SingleFileSnapshotInfo> allSnapshotFiles = new ArrayList<>();
    try (DirectoryStream<Path> stream =
             Files.newDirectoryStream(smDir.toPath())) {
      for (Path path : stream) {
        Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
        if (matcher.matches()) {
          final long endIndex = Long.parseLong(matcher.group(2));
          final long term = Long.parseLong(matcher.group(1));
          final FileInfo fileInfo = new FileInfo(path, null); //We don't need FileDigest here.
          allSnapshotFiles.add(new SingleFileSnapshotInfo(fileInfo, term, endIndex));
        }
      }
    }

    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
      allSnapshotFiles.sort(new SnapshotFileComparator());
      List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
          snapshotRetentionPolicy.getNumSnapshotsRetained(), allSnapshotFiles.size()).stream()
          .map(singleFileSnapshotInfo -> singleFileSnapshotInfo.getFile().getPath().toFile())
          .collect(Collectors.toList());
      for (File snapshotFile : snapshotFilesToBeCleaned) {
        LOG.info("Deleting old snapshot at {}", snapshotFile.getAbsolutePath());
        FileUtils.deleteFileQuietly(snapshotFile);
      }
    }
  }
}
 
Example 8
Source Project: incubator-ratis   Source File: RaftStorageDirectory.java    License: Apache License 2.0 5 votes vote down vote up
private static void clearDirectory(File dir) throws IOException {
  if (dir.exists()) {
    LOG.info(dir + " already exists.  Deleting it ...");
    FileUtils.deleteFully(dir);
  }
  FileUtils.createDirectories(dir);
}
 
Example 9
Source Project: incubator-ratis   Source File: RaftStorageDirectory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check consistency of the storage directory.
 *
 * @return state {@link StorageState} of the storage directory
 */
StorageState analyzeStorage(boolean toLock) throws IOException {
  Objects.requireNonNull(root, "root directory is null");

  String rootPath = root.getCanonicalPath();
  try { // check that storage exists
    if (!root.exists()) {
      LOG.info("The storage directory " + rootPath + " does not exist. Creating ...");
      FileUtils.createDirectories(root);
    }
    // or is inaccessible
    if (!root.isDirectory()) {
      LOG.warn(rootPath + " is not a directory");
      return StorageState.NON_EXISTENT;
    }
    if (!Files.isWritable(root.toPath())) {
      LOG.warn("The storage directory " + rootPath + " is not writable.");
      return StorageState.NON_EXISTENT;
    }
  } catch(SecurityException ex) {
    LOG.warn("Cannot access storage directory " + rootPath, ex);
    return StorageState.NON_EXISTENT;
  }

  if (toLock) {
    this.lock(); // lock storage if it exists
  }

  // check whether current directory is valid
  if (hasMetaFile()) {
    return StorageState.NORMAL;
  } else {
    return StorageState.NOT_FORMATTED;
  }
}
 
Example 10
Source Project: incubator-ratis   Source File: RaftStorageDirectory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Lock storage to provide exclusive access.
 *
 * <p> Locking is not supported by all file systems.
 * E.g., NFS does not consistently support exclusive locks.
 *
 * <p> If locking is supported we guarantee exclusive access to the
 * storage directory. Otherwise, no guarantee is given.
 *
 * @throws IOException if locking fails
 */
public void lock() throws IOException {
  final File lockF = new File(root, STORAGE_FILE_LOCK);
  final FileLock newLock = FileUtils.attempt(() -> tryLock(lockF), () -> "tryLock " + lockF);
  if (newLock == null) {
    String msg = "Cannot lock storage " + this.root
        + ". The directory is already locked";
    LOG.info(msg);
    throw new IOException(msg);
  }
  // Don't overwrite lock until success - this way if we accidentally
  // call lock twice, the internal state won't be cleared by the second
  // (failed) lock attempt
  lock = newLock;
}
 
Example 11
Source Project: incubator-ratis   Source File: MiniRaftCluster.java    License: Apache License 2.0 5 votes vote down vote up
private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDir(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 12
Source Project: ratis   Source File: TestRaftStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * check if RaftStorage deletes tmp metafile when startup
 */
@Test
public void testCleanMetaTmpFile() throws Exception {
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  Assert.assertEquals(StorageState.NORMAL, storage.getState());
  storage.close();

  RaftStorageDirectory sd = new RaftStorageDirectory(storageDir);
  File metaFile = sd.getMetaFile();
  FileUtils.move(metaFile, sd.getMetaTmpFile());

  Assert.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false));

  try {
    new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.fail("should throw IOException since storage dir is not formatted");
  } catch (IOException e) {
    Assert.assertTrue(
        e.getMessage().contains(StorageState.NOT_FORMATTED.name()));
  }

  // let the storage dir contain both raft-meta and raft-meta.tmp
  new RaftStorage(storageDir, StartupOption.FORMAT).close();
  Assert.assertTrue(sd.getMetaFile().exists());
  Assert.assertTrue(sd.getMetaTmpFile().createNewFile());
  Assert.assertTrue(sd.getMetaTmpFile().exists());
  try {
    storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.assertEquals(StorageState.NORMAL, storage.getState());
    Assert.assertFalse(sd.getMetaTmpFile().exists());
    Assert.assertTrue(sd.getMetaFile().exists());
  } finally {
    storage.close();
  }
}
 
Example 13
Source Project: ratis   Source File: TestRaftLogSegment.java    License: Apache License 2.0 5 votes vote down vote up
File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten)
    throws IOException {
  if (!isOpen) {
    Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written.");
  }
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  final File file = isOpen ?
      storage.getStorageDir().getOpenLogFile(startIndex) :
      storage.getStorageDir().getClosedLogFile(startIndex, startIndex + numEntries - 1);

  final LogEntryProto[] entries = new LogEntryProto[numEntries];
  try (LogOutputStream out = new LogOutputStream(file, false,
      segmentMaxSize, preallocatedSize, bufferSize)) {
    for (int i = 0; i < entries.length; i++) {
      SimpleOperation op = new SimpleOperation("m" + i);
      entries[i] = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + startIndex);
      out.write(entries[i]);
    }
  }

  if (isLastEntryPartiallyWritten) {
    final int entrySize = size(entries[entries.length - 1]);
    final int truncatedEntrySize = ThreadLocalRandom.current().nextInt(entrySize - 1) + 1;
    // 0 < truncatedEntrySize < entrySize
    final long fileLength = file.length();
    final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize);
    LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})",
        entrySize, truncatedEntrySize, fileLength, truncatedFileLength);
    FileUtils.truncateFile(file, truncatedFileLength);
  }

  storage.close();
  return file;
}
 
Example 14
Source Project: ratis   Source File: FileStoreStateMachine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage)
    throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  FileUtils.createDirectories(files.getRoot());
}
 
Example 15
Source Project: ratis   Source File: FileStore.java    License: Apache License 2.0 5 votes vote down vote up
CompletableFuture<Path> delete(long index, String relative) {
  final Supplier<String> name = () -> "delete(" + relative + ") @" + getId() + ":" + index;
  final CheckedSupplier<Path, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.remove(relative);
    FileUtils.delete(resolve(info.getRelativePath()));
    return info.getRelativePath();
  }, name);
  return submit(task, deleter);
}
 
Example 16
Source Project: ratis   Source File: LogSegment.java    License: Apache License 2.0 5 votes vote down vote up
static LogSegment loadSegment(RaftStorage storage, File file,
    long start, long end, boolean isOpen,
    boolean keepEntryInCache, Consumer<LogEntryProto> logConsumer)
    throws IOException {
  final LogSegment segment = isOpen ?
      LogSegment.newOpenSegment(storage, start) :
      LogSegment.newCloseSegment(storage, start, end);

  final int entryCount = readSegmentFile(file, start, end, isOpen, entry -> {
    segment.append(keepEntryInCache || isOpen, entry);
    if (logConsumer != null) {
      logConsumer.accept(entry);
    }
  });
  LOG.info("Successfully read {} entries from segment file {}", entryCount, file);

  if (entryCount == 0) {
    // The segment does not have any entries, delete the file.
    FileUtils.deleteFile(file);
    return null;
  } else if (file.length() > segment.getTotalSize()) {
    // The segment has extra padding, truncate it.
    FileUtils.truncateFile(file, segment.getTotalSize());
  }

  Preconditions.assertTrue(start == segment.getStartIndex());
  if (!segment.records.isEmpty()) {
    Preconditions.assertTrue(start == segment.records.get(0).getTermIndex().getIndex());
  }
  if (!isOpen) {
    Preconditions.assertTrue(segment.getEndIndex() == end);
  }
  return segment;
}
 
Example 17
Source Project: ratis   Source File: LogOutputStream.java    License: Apache License 2.0 5 votes vote down vote up
public LogOutputStream(File file, boolean append, long segmentMaxSize,
    long preallocatedSize, int bufferSize)
    throws IOException {
  this.file = file;
  this.checksum = new PureJavaCrc32C();
  this.segmentMaxSize = segmentMaxSize;
  this.preallocatedSize = preallocatedSize;
  RandomAccessFile rp = new RandomAccessFile(file, "rw");
  fc = rp.getChannel();
  fc.position(fc.size());
  preallocatedPos = fc.size();
  out = new BufferedWriteChannel(fc, bufferSize);

  try {
    fc = rp.getChannel();
    fc.position(fc.size());
    preallocatedPos = fc.size();

    out = new BufferedWriteChannel(fc, bufferSize);
    if (!append) {
      create();
    }
  } catch (IOException ioe) {
    LOG.warn("Hit IOException while creating log segment " + file
        + ", delete the partial file.");
    // hit IOException, clean up the in-progress log file
    try {
      FileUtils.deleteFully(file);
    } catch (IOException e) {
      LOG.warn("Failed to delete the file " + file, e);
    }
    throw ioe;
  }
}
 
Example 18
Source Project: ratis   Source File: RaftStorageDirectory.java    License: Apache License 2.0 5 votes vote down vote up
private static void clearDirectory(File dir) throws IOException {
  if (dir.exists()) {
    LOG.info(dir + " already exists.  Deleting it ...");
    FileUtils.deleteFully(dir);
  }
  FileUtils.createDirectories(dir);
}
 
Example 19
Source Project: ratis   Source File: RaftStorageDirectory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check consistency of the storage directory.
 *
 * @return state {@link StorageState} of the storage directory
 */
StorageState analyzeStorage(boolean toLock) throws IOException {
  Objects.requireNonNull(root, "root directory is null");

  String rootPath = root.getCanonicalPath();
  try { // check that storage exists
    if (!root.exists()) {
      LOG.info("The storage directory " + rootPath + " does not exist. Creating ...");
      FileUtils.createDirectories(root);
    }
    // or is inaccessible
    if (!root.isDirectory()) {
      LOG.warn(rootPath + " is not a directory");
      return StorageState.NON_EXISTENT;
    }
    if (!Files.isWritable(root.toPath())) {
      LOG.warn("The storage directory " + rootPath + " is not writable.");
      return StorageState.NON_EXISTENT;
    }
  } catch(SecurityException ex) {
    LOG.warn("Cannot access storage directory " + rootPath, ex);
    return StorageState.NON_EXISTENT;
  }

  if (toLock) {
    this.lock(); // lock storage if it exists
  }

  // check whether current directory is valid
  if (hasMetaFile()) {
    return StorageState.NORMAL;
  } else {
    return StorageState.NOT_FORMATTED;
  }
}
 
Example 20
Source Project: ratis   Source File: MiniRaftCluster.java    License: Apache License 2.0 5 votes vote down vote up
private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDirs(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 21
Source Project: ratis   Source File: ServerRestartTests.java    License: Apache License 2.0 5 votes vote down vote up
static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception {
  // truncate log
  FileUtils.truncateFile(openLogFile, openLogFile.length() - 1);
  final RaftServerImpl server = cluster.restartServer(id, false);
  // the last index should be one less than before
  Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex());
  server.getProxy().close();
}
 
Example 22
Source Project: hadoop-ozone   Source File: TestContainerMapper.java    License: Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void shutdown() throws IOException {
  cluster.shutdown();
  FileUtils.deleteFully(new File(dbPath));
}
 
Example 23
Source Project: incubator-ratis   Source File: TestLogSegment.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 
Example 24
Source Project: incubator-ratis   Source File: TestSegmentedRaftLog.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 
Example 25
Source Project: incubator-ratis   Source File: TestRaftLogReadWrite.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 
Example 26
Source Project: incubator-ratis   Source File: TestServerState.java    License: Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void tearDown() throws IOException {
  FileUtils.deleteFully(rootTestDir.get());
}
 
Example 27
Source Project: incubator-ratis   Source File: TestRaftStorage.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 
Example 28
Source Project: incubator-ratis   Source File: TestRaftServerConfigKeys.java    License: Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void tearDown() throws IOException {
  FileUtils.deleteFully(rootTestDir.get());
}
 
Example 29
Source Project: incubator-ratis   Source File: FileInfo.java    License: Apache License 2.0 4 votes vote down vote up
FileOut(Path p) throws IOException {
  this.out = FileUtils.createNewFile(p);
  this.channel = Channels.newChannel(out);
}
 
Example 30
Source Project: incubator-ratis   Source File: MetadataServer.java    License: Apache License 2.0 4 votes vote down vote up
public void cleanUp() throws IOException {
    FileUtils.deleteFully(new File(getServerOpts().getWorkingDir()));
}