org.apache.ratis.server.storage.FileInfo Java Examples

The following examples show how to use org.apache.ratis.server.storage.FileInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleStateMachineStorage.java    From incubator-ratis with Apache License 2.0 6 votes vote down vote up
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 
Example #2
Source File: SimpleStateMachineStorage.java    From ratis with Apache License 2.0 6 votes vote down vote up
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 
Example #3
Source File: LogAppender.java    From ratis with Apache License 2.0 5 votes vote down vote up
private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 
Example #4
Source File: SimpleStateMachineStorage.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
@Override
public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) throws IOException {
  if (snapshotRetentionPolicy != null && snapshotRetentionPolicy.getNumSnapshotsRetained() > 0) {

    List<SingleFileSnapshotInfo> allSnapshotFiles = new ArrayList<>();
    try (DirectoryStream<Path> stream =
             Files.newDirectoryStream(smDir.toPath())) {
      for (Path path : stream) {
        Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
        if (matcher.matches()) {
          final long endIndex = Long.parseLong(matcher.group(2));
          final long term = Long.parseLong(matcher.group(1));
          final FileInfo fileInfo = new FileInfo(path, null); //We don't need FileDigest here.
          allSnapshotFiles.add(new SingleFileSnapshotInfo(fileInfo, term, endIndex));
        }
      }
    }

    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
      allSnapshotFiles.sort(new SnapshotFileComparator());
      List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
          snapshotRetentionPolicy.getNumSnapshotsRetained(), allSnapshotFiles.size()).stream()
          .map(singleFileSnapshotInfo -> singleFileSnapshotInfo.getFile().getPath().toFile())
          .collect(Collectors.toList());
      for (File snapshotFile : snapshotFilesToBeCleaned) {
        LOG.info("Deleting old snapshot at {}", snapshotFile.getAbsolutePath());
        FileUtils.deleteFileQuietly(snapshotFile);
      }
    }
  }
}
 
Example #5
Source File: LogAppender.java    From incubator-ratis with Apache License 2.0 5 votes vote down vote up
private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 
Example #6
Source File: SingleFileSnapshotInfo.java    From incubator-ratis with Apache License 2.0 4 votes vote down vote up
/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 
Example #7
Source File: SingleFileSnapshotInfo.java    From ratis with Apache License 2.0 4 votes vote down vote up
/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 
Example #8
Source File: SingleFileSnapshotInfo.java    From ratis with Apache License 2.0 4 votes vote down vote up
public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 
Example #9
Source File: FileListSnapshotInfo.java    From ratis with Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return files;
}
 
Example #10
Source File: FileListSnapshotInfo.java    From ratis with Apache License 2.0 4 votes vote down vote up
public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 
Example #11
Source File: TestContainerStateMachineFailures.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
@Test
public void testApplyTransactionFailure() throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.
          getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  KeyValueContainerData keyValueContainerData =
          (KeyValueContainerData) containerData;
  key.close();
  ContainerStateMachine stateMachine =
      (ContainerStateMachine) TestHelper.getStateMachine(cluster.
          getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  stateMachine.takeSnapshot();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  // Since the snapshot threshold is set to 1, since there are
  // applyTransactions, we should see snapshots
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  // delete the container db file
  FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  // close container transaction will fail over Ratis and will initiate
  // a pipeline close action

  try {
    xceiverClient.sendCommand(request.build());
    Assert.fail("Expected exception not thrown");
  } catch (IOException e) {
    // Exception should be thrown
  }
  // Make sure the container is marked unhealthy
  Assert.assertTrue(dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
  try {
    // try to take a new snapshot, ideally it should just fail
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.assertTrue(ioe instanceof StateMachineException);
  }
  // Make sure the latest snapshot is same as the previous one
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 
Example #12
Source File: SingleFileSnapshotInfo.java    From incubator-ratis with Apache License 2.0 4 votes vote down vote up
public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 
Example #13
Source File: FileListSnapshotInfo.java    From incubator-ratis with Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return files;
}
 
Example #14
Source File: FileListSnapshotInfo.java    From incubator-ratis with Apache License 2.0 4 votes vote down vote up
public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 
Example #15
Source File: OMRatisSnapshotInfo.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
@Override
public List<FileInfo> getFiles() {
  return null;
}
 
Example #16
Source File: TestContainerStateMachineFailures.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
@Test
public void testApplyTransactionIdempotencyWithClosedContainer()
        throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  key.close();
  ContainerStateMachine stateMachine =
          (ContainerStateMachine) TestHelper.getStateMachine(dn,
                  omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  stateMachine.takeSnapshot();
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  try {
    xceiverClient.sendCommand(request.build());
  } catch (IOException e) {
    Assert.fail("Exception should not be thrown");
  }
  Assert.assertTrue(
          TestHelper.getDatanodeService(omKeyLocationInfo, cluster)
                  .getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.CLOSED);
  Assert.assertTrue(stateMachine.isStateMachineHealthy());
  try {
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.fail("Exception should not be thrown");
  }
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 
Example #17
Source File: SnapshotInfo.java    From ratis with Apache License 2.0 2 votes vote down vote up
/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();
 
Example #18
Source File: SnapshotInfo.java    From incubator-ratis with Apache License 2.0 2 votes vote down vote up
/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();