Java Code Examples for org.apache.ratis.server.storage.FileInfo

The following examples show how to use org.apache.ratis.server.storage.FileInfo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-ratis   Source File: SimpleStateMachineStorage.java    License: Apache License 2.0 6 votes vote down vote up
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 
Example 2
Source Project: ratis   Source File: SimpleStateMachineStorage.java    License: Apache License 2.0 6 votes vote down vote up
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 
Example 3
Source Project: incubator-ratis   Source File: SimpleStateMachineStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) throws IOException {
  if (snapshotRetentionPolicy != null && snapshotRetentionPolicy.getNumSnapshotsRetained() > 0) {

    List<SingleFileSnapshotInfo> allSnapshotFiles = new ArrayList<>();
    try (DirectoryStream<Path> stream =
             Files.newDirectoryStream(smDir.toPath())) {
      for (Path path : stream) {
        Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
        if (matcher.matches()) {
          final long endIndex = Long.parseLong(matcher.group(2));
          final long term = Long.parseLong(matcher.group(1));
          final FileInfo fileInfo = new FileInfo(path, null); //We don't need FileDigest here.
          allSnapshotFiles.add(new SingleFileSnapshotInfo(fileInfo, term, endIndex));
        }
      }
    }

    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
      allSnapshotFiles.sort(new SnapshotFileComparator());
      List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
          snapshotRetentionPolicy.getNumSnapshotsRetained(), allSnapshotFiles.size()).stream()
          .map(singleFileSnapshotInfo -> singleFileSnapshotInfo.getFile().getPath().toFile())
          .collect(Collectors.toList());
      for (File snapshotFile : snapshotFilesToBeCleaned) {
        LOG.info("Deleting old snapshot at {}", snapshotFile.getAbsolutePath());
        FileUtils.deleteFileQuietly(snapshotFile);
      }
    }
  }
}
 
Example 4
Source Project: incubator-ratis   Source File: LogAppender.java    License: Apache License 2.0 5 votes vote down vote up
private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 
Example 5
Source Project: ratis   Source File: LogAppender.java    License: Apache License 2.0 5 votes vote down vote up
private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 
Example 6
@Test
public void testApplyTransactionFailure() throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.
          getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  KeyValueContainerData keyValueContainerData =
          (KeyValueContainerData) containerData;
  key.close();
  ContainerStateMachine stateMachine =
      (ContainerStateMachine) TestHelper.getStateMachine(cluster.
          getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  stateMachine.takeSnapshot();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  // Since the snapshot threshold is set to 1, since there are
  // applyTransactions, we should see snapshots
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  // delete the container db file
  FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  // close container transaction will fail over Ratis and will initiate
  // a pipeline close action

  try {
    xceiverClient.sendCommand(request.build());
    Assert.fail("Expected exception not thrown");
  } catch (IOException e) {
    // Exception should be thrown
  }
  // Make sure the container is marked unhealthy
  Assert.assertTrue(dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
  try {
    // try to take a new snapshot, ideally it should just fail
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.assertTrue(ioe instanceof StateMachineException);
  }
  // Make sure the latest snapshot is same as the previous one
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 
Example 7
@Test
public void testApplyTransactionIdempotencyWithClosedContainer()
        throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  key.close();
  ContainerStateMachine stateMachine =
          (ContainerStateMachine) TestHelper.getStateMachine(dn,
                  omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  stateMachine.takeSnapshot();
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  try {
    xceiverClient.sendCommand(request.build());
  } catch (IOException e) {
    Assert.fail("Exception should not be thrown");
  }
  Assert.assertTrue(
          TestHelper.getDatanodeService(omKeyLocationInfo, cluster)
                  .getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.CLOSED);
  Assert.assertTrue(stateMachine.isStateMachineHealthy());
  try {
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.fail("Exception should not be thrown");
  }
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 
Example 8
Source Project: hadoop-ozone   Source File: OMRatisSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return null;
}
 
Example 9
Source Project: incubator-ratis   Source File: FileListSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 
Example 10
Source Project: incubator-ratis   Source File: FileListSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return files;
}
 
Example 11
Source Project: incubator-ratis   Source File: SingleFileSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 
Example 12
Source Project: incubator-ratis   Source File: SingleFileSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 
Example 13
Source Project: ratis   Source File: FileListSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 
Example 14
Source Project: ratis   Source File: FileListSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return files;
}
 
Example 15
Source Project: ratis   Source File: SingleFileSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 
Example 16
Source Project: ratis   Source File: SingleFileSnapshotInfo.java    License: Apache License 2.0 4 votes vote down vote up
/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 
Example 17
Source Project: incubator-ratis   Source File: SnapshotInfo.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();
 
Example 18
Source Project: ratis   Source File: SnapshotInfo.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();