org.apache.ratis.statemachine.impl.SimpleStateMachineStorage Java Examples
The following examples show how to use
org.apache.ratis.statemachine.impl.SimpleStateMachineStorage.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ContainerStateMachine.java From hadoop-ozone with Apache License 2.0 | 6 votes |
private long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { if (snapshot == null) { TermIndex empty = TermIndex.newTermIndex(0, RaftLog.INVALID_LOG_INDEX); LOG.info("{}: The snapshot info is null. Setting the last applied index" + "to:{}", gid, empty); setLastAppliedTermIndex(empty); return empty.getIndex(); } final File snapshotFile = snapshot.getFile().getPath().toFile(); final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); LOG.info("{}: Setting the last applied index to {}", gid, last); setLastAppliedTermIndex(last); // initialize the dispatcher with snapshot so that it build the missing // container list buildMissingContainerSet(snapshotFile); return last.getIndex(); }
Example #2
Source File: TestRaftStorage.java From incubator-ratis with Apache License 2.0 | 6 votes |
@Test public void testSnapshotFileName() throws Exception { final long term = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); final long index = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); final String name = SimpleStateMachineStorage.getSnapshotFileName(term, index); System.out.println("name = " + name); final File file = new File(storageDir, name); final TermIndex ti = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(file); System.out.println("file = " + file); Assert.assertEquals(term, ti.getTerm()); Assert.assertEquals(index, ti.getIndex()); System.out.println("ti = " + ti); final File foo = new File(storageDir, "foo"); try { SimpleStateMachineStorage.getTermIndexFromSnapshotFile(foo); Assert.fail(); } catch(IllegalArgumentException iae) { System.out.println("Good " + iae); } }
Example #3
Source File: ArithmeticStateMachine.java From incubator-ratis with Apache License 2.0 | 6 votes |
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(AutoCloseableLock writeLock = writeLock(); ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); variables.putAll(JavaUtils.cast(in.readObject())); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
Example #4
Source File: TestRaftStorage.java From ratis with Apache License 2.0 | 6 votes |
@Test public void testSnapshotFileName() throws Exception { final long term = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); final long index = ThreadLocalRandom.current().nextLong(Long.MAX_VALUE); final String name = SimpleStateMachineStorage.getSnapshotFileName(term, index); System.out.println("name = " + name); final File file = new File(storageDir, name); final TermIndex ti = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(file); System.out.println("file = " + file); Assert.assertEquals(term, ti.getTerm()); Assert.assertEquals(index, ti.getIndex()); System.out.println("ti = " + ti); final File foo = new File(storageDir, "foo"); try { SimpleStateMachineStorage.getTermIndexFromSnapshotFile(foo); Assert.fail(); } catch(IllegalArgumentException iae) { System.out.println("Good " + iae); } }
Example #5
Source File: ArithmeticStateMachine.java From ratis with Apache License 2.0 | 6 votes |
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(final AutoCloseableLock writeLock = writeLock(); final ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); variables.putAll(JavaUtils.cast(in.readObject())); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
Example #6
Source File: LogStateMachine.java From ratis with Apache License 2.0 | 6 votes |
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(final AutoCloseableLock writeLock = writeLock(); final ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); this.length = in.readLong(); this.state = (State) in.readObject(); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
Example #7
Source File: TestFreonWithDatanodeFastRestart.java From hadoop-ozone with Apache License 2.0 | 5 votes |
@Test @Ignore("TODO:HDDS-1160") public void testRestart() throws Exception { startFreon(); StateMachine sm = getStateMachine(); TermIndex termIndexBeforeRestart = sm.getLastAppliedTermIndex(); cluster.restartHddsDatanode(0, false); sm = getStateMachine(); SimpleStateMachineStorage storage = (SimpleStateMachineStorage)sm.getStateMachineStorage(); SingleFileSnapshotInfo snapshotInfo = storage.getLatestSnapshot(); TermIndex termInSnapshot = snapshotInfo.getTermIndex(); String expectedSnapFile = storage.getSnapshotFile(termIndexBeforeRestart.getTerm(), termIndexBeforeRestart.getIndex()).getAbsolutePath(); Assert.assertEquals(expectedSnapFile, snapshotInfo.getFile().getPath().toString()); Assert.assertEquals(termInSnapshot, termIndexBeforeRestart); // After restart the term index might have progressed to apply pending // transactions. TermIndex termIndexAfterRestart = sm.getLastAppliedTermIndex(); Assert.assertTrue(termIndexAfterRestart.getIndex() >= termIndexBeforeRestart.getIndex()); // TODO: fix me // Give some time for the datanode to register again with SCM. // If we try to use the pipeline before the datanode registers with SCM // we end up in "NullPointerException: scmId cannot be null" in // datanode statemachine and datanode crashes. // This has to be fixed. Check HDDS-830. // Until then this sleep should help us! Thread.sleep(5000); startFreon(); }
Example #8
Source File: CounterStateMachine.java From incubator-ratis with Apache License 2.0 | 5 votes |
/** * Load the state of the state machine from the storage. * * @param snapshot to load * @return the index of the snapshot or -1 if snapshot is invalid * @throws IOException if any error happens during read from storage */ private long load(SingleFileSnapshotInfo snapshot) throws IOException { //check the snapshot nullity if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftLog.INVALID_LOG_INDEX; } //check the existance of the snapshot file final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftLog.INVALID_LOG_INDEX; } //load the TermIndex object for the snapshot using the file name pattern of // the snapshot final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); //read the file and cast it to the AtomicInteger and set the counter try (ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { //set the last applied termIndex to the termIndex of the snapshot setLastAppliedTermIndex(last); //read, cast and set the counter counter = JavaUtils.cast(in.readObject()); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
Example #9
Source File: LogStateMachine.java From incubator-ratis with Apache License 2.0 | 5 votes |
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(AutoCloseableLock writeLock = writeLock(); ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); this.length = in.readLong(); this.dataRecordsSize = in.readLong(); this.state = (State) in.readObject(); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
Example #10
Source File: RaftSnapshotBaseTest.java From incubator-ratis with Apache License 2.0 | 5 votes |
public static List<File> getSnapshotFiles(MiniRaftCluster cluster, long startIndex, long endIndex) { final RaftServerImpl leader = cluster.getLeader(); final SimpleStateMachineStorage storage = SimpleStateMachine4Testing.get(leader).getStateMachineStorage(); final long term = leader.getState().getCurrentTerm(); return LongStream.range(startIndex, endIndex) .mapToObj(i -> storage.getSnapshotFile(term, i)) .collect(Collectors.toList()); }
Example #11
Source File: RaftSnapshotBaseTest.java From ratis with Apache License 2.0 | 5 votes |
static List<File> getSnapshotFiles(MiniRaftCluster cluster, long startIndex, long endIndex) { final RaftServerImpl leader = cluster.getLeader(); final SimpleStateMachineStorage storage = SimpleStateMachine4Testing.get(leader).getStateMachineStorage(); final long term = leader.getState().getCurrentTerm(); return LongStream.range(startIndex, endIndex) .mapToObj(i -> storage.getSnapshotFile(term, i)) .collect(Collectors.toList()); }
Example #12
Source File: TestContainerStateMachineFailures.java From hadoop-ozone with Apache License 2.0 | 4 votes |
@Test public void testApplyTransactionFailure() throws Exception { OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName) .createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>()); // First write and flush creates a container in the datanode key.write("ratis".getBytes()); key.flush(); key.write("ratis".getBytes()); KeyOutputStream groupOutputStream = (KeyOutputStream) key. getOutputStream(); List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList(); Assert.assertEquals(1, locationInfoList.size()); OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0); HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, cluster); int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails()); ContainerData containerData = dn.getDatanodeStateMachine() .getContainer().getContainerSet() .getContainer(omKeyLocationInfo.getContainerID()) .getContainerData(); Assert.assertTrue(containerData instanceof KeyValueContainerData); KeyValueContainerData keyValueContainerData = (KeyValueContainerData) containerData; key.close(); ContainerStateMachine stateMachine = (ContainerStateMachine) TestHelper.getStateMachine(cluster. getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline()); SimpleStateMachineStorage storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage(); stateMachine.takeSnapshot(); Path parentPath = storage.findLatestSnapshot().getFile().getPath(); // Since the snapshot threshold is set to 1, since there are // applyTransactions, we should see snapshots Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0); FileInfo snapshot = storage.findLatestSnapshot().getFile(); Assert.assertNotNull(snapshot); long containerID = omKeyLocationInfo.getContainerID(); // delete the container db file FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath())); Pipeline pipeline = cluster.getStorageContainerLocationClient() .getContainerWithPipeline(containerID).getPipeline(); XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline); ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setDatanodeUuid(pipeline.getFirstNode().getUuidString()); request.setCmdType(ContainerProtos.Type.CloseContainer); request.setContainerID(containerID); request.setCloseContainer( ContainerProtos.CloseContainerRequestProto.getDefaultInstance()); // close container transaction will fail over Ratis and will initiate // a pipeline close action try { xceiverClient.sendCommand(request.build()); Assert.fail("Expected exception not thrown"); } catch (IOException e) { // Exception should be thrown } // Make sure the container is marked unhealthy Assert.assertTrue(dn.getDatanodeStateMachine() .getContainer().getContainerSet().getContainer(containerID) .getContainerState() == ContainerProtos.ContainerDataProto.State.UNHEALTHY); try { // try to take a new snapshot, ideally it should just fail stateMachine.takeSnapshot(); } catch (IOException ioe) { Assert.assertTrue(ioe instanceof StateMachineException); } // Make sure the latest snapshot is same as the previous one FileInfo latestSnapshot = storage.findLatestSnapshot().getFile(); Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath())); }
Example #13
Source File: TestContainerStateMachineFailures.java From hadoop-ozone with Apache License 2.0 | 4 votes |
@Test public void testApplyTransactionIdempotencyWithClosedContainer() throws Exception { OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName) .createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>()); // First write and flush creates a container in the datanode key.write("ratis".getBytes()); key.flush(); key.write("ratis".getBytes()); KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream(); List<OmKeyLocationInfo> locationInfoList = groupOutputStream.getLocationInfoList(); Assert.assertEquals(1, locationInfoList.size()); OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0); HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, cluster); ContainerData containerData = dn.getDatanodeStateMachine() .getContainer().getContainerSet() .getContainer(omKeyLocationInfo.getContainerID()) .getContainerData(); Assert.assertTrue(containerData instanceof KeyValueContainerData); key.close(); ContainerStateMachine stateMachine = (ContainerStateMachine) TestHelper.getStateMachine(dn, omKeyLocationInfo.getPipeline()); SimpleStateMachineStorage storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage(); Path parentPath = storage.findLatestSnapshot().getFile().getPath(); stateMachine.takeSnapshot(); Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0); FileInfo snapshot = storage.findLatestSnapshot().getFile(); Assert.assertNotNull(snapshot); long containerID = omKeyLocationInfo.getContainerID(); Pipeline pipeline = cluster.getStorageContainerLocationClient() .getContainerWithPipeline(containerID).getPipeline(); XceiverClientSpi xceiverClient = xceiverClientManager.acquireClient(pipeline); ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setDatanodeUuid(pipeline.getFirstNode().getUuidString()); request.setCmdType(ContainerProtos.Type.CloseContainer); request.setContainerID(containerID); request.setCloseContainer( ContainerProtos.CloseContainerRequestProto.getDefaultInstance()); try { xceiverClient.sendCommand(request.build()); } catch (IOException e) { Assert.fail("Exception should not be thrown"); } Assert.assertTrue( TestHelper.getDatanodeService(omKeyLocationInfo, cluster) .getDatanodeStateMachine() .getContainer().getContainerSet().getContainer(containerID) .getContainerState() == ContainerProtos.ContainerDataProto.State.CLOSED); Assert.assertTrue(stateMachine.isStateMachineHealthy()); try { stateMachine.takeSnapshot(); } catch (IOException ioe) { Assert.fail("Exception should not be thrown"); } FileInfo latestSnapshot = storage.findLatestSnapshot().getFile(); Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath())); }
Example #14
Source File: SimpleStateMachine4Testing.java From incubator-ratis with Apache License 2.0 | 4 votes |
@Override public SimpleStateMachineStorage getStateMachineStorage() { return storage; }
Example #15
Source File: SimpleStateMachine4Testing.java From ratis with Apache License 2.0 | 4 votes |
@Override public SimpleStateMachineStorage getStateMachineStorage() { return storage; }