com.alipay.sofa.jraft.storage.snapshot.SnapshotReader Java Examples

The following examples show how to use com.alipay.sofa.jraft.storage.snapshot.SnapshotReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: sofa-jraft   Author: sofastack   File: AtomicStateMachine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    if (isLeader()) {
        LOG.warn("Leader is not supposed to load snapshot");
        return false;
    }
    if (reader.getFileMeta("data") == null) {
        LOG.error("Fail to find data file in {}", reader.getPath());
        return false;
    }
    final AtomicSnapshotFile snapshot = new AtomicSnapshotFile(reader.getPath() + File.separator + "data");
    try {
        final Map<String, Long> values = snapshot.load();
        this.counters.clear();
        if (values != null) {
            for (final Map.Entry<String, Long> entry : values.entrySet()) {
                this.counters.put(entry.getKey(), new AtomicLong(entry.getValue()));
            }
        }
        return true;
    } catch (final IOException e) {
        LOG.error("Fail to load snapshot from {}", snapshot.getPath());
        return false;
    }

}
 
Example #2
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public SnapshotReader open() {
    long lsIndex = 0;
    this.lock.lock();
    try {
        if (this.lastSnapshotIndex != 0) {
            lsIndex = this.lastSnapshotIndex;
            ref(lsIndex);
        }
    } finally {
        this.lock.unlock();
    }
    if (lsIndex == 0) {
        LOG.warn("No data for snapshot reader {}.", this.path);
        return null;
    }
    final String snapshotPath = getSnapshotPath(lsIndex);
    final SnapshotReader reader = new LocalSnapshotReader(this, this.snapshotThrottle, this.addr, this.raftOptions,
        snapshotPath);
    if (!reader.init(null)) {
        LOG.error("Fail to init reader for path {}.", snapshotPath);
        unref(lsIndex);
        return null;
    }
    return reader;
}
 
Example #3
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public SnapshotReader copyFrom(final String uri, final SnapshotCopierOptions opts) {
    final SnapshotCopier copier = startToCopyFrom(uri, opts);
    if (copier == null) {
        return null;
    }
    try {
        copier.join();
    } catch (final InterruptedException e) {
        Thread.currentThread().interrupt();
        LOG.error("Join on snapshot copier was interrupted.");
        return null;
    }
    final SnapshotReader reader = copier.getReader();
    Utils.closeQuietly(copier);
    return reader;
}
 
Example #4
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotCopierTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testCancelByRemote() throws Exception {
    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
        .setReadPartly(true);

    //mock get metadata
    final ArgumentCaptor<RpcResponseClosure> argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
    this.copier.start();
    Thread.sleep(500);
    final RpcResponseClosure<RpcRequests.GetFileResponse> closure = argument.getValue();

    closure.run(new Status(RaftError.ECANCELED, "test cancel"));

    this.copier.join();
    //start timer
    final SnapshotReader reader = this.copier.getReader();
    assertNull(reader);
    Assert.assertEquals(RaftError.ECANCELED.getNumber(), this.copier.getCode());
    Assert.assertEquals("test cancel", this.copier.getErrorMsg());
}
 
Example #5
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateOpen() throws Exception {
    SnapshotWriter writer = this.snapshotStorage.create();
    assertNotNull(writer);
    RaftOutter.SnapshotMeta wroteMeta = RaftOutter.SnapshotMeta.newBuilder()
        .setLastIncludedIndex(this.lastSnapshotIndex + 1).setLastIncludedTerm(1).build();
    ((LocalSnapshotWriter) writer).saveMeta(wroteMeta);
    writer.addFile("data");
    assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex).get());
    writer.close();
    //release old
    assertEquals(0, this.snapshotStorage.getRefs(this.lastSnapshotIndex).get());
    //ref new
    assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
    SnapshotReader reader = this.snapshotStorage.open();
    assertNotNull(reader);
    assertTrue(reader.listFiles().contains("data"));
    RaftOutter.SnapshotMeta readMeta = reader.load();
    assertEquals(wroteMeta, readMeta);
    assertEquals(2, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
    reader.close();
    assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
}
 
Example #6
Source Project: sofa-jraft   Author: sofastack   File: FSMCallerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnSnapshotLoad() throws Exception {
    final SnapshotReader reader = Mockito.mock(SnapshotReader.class);

    final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
    Mockito.when(reader.load()).thenReturn(meta);
    Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(true);
    final CountDownLatch latch = new CountDownLatch(1);
    this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {

        @Override
        public void run(final Status status) {
            assertTrue(status.isOk());
            latch.countDown();
        }

        @Override
        public SnapshotReader start() {
            return reader;
        }
    });
    latch.await();
    assertEquals(this.fsmCaller.getLastAppliedIndex(), 12);
    Mockito.verify(this.fsm).onConfigurationCommitted(Mockito.any());
}
 
Example #7
Source Project: sofa-jraft   Author: sofastack   File: FSMCallerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnSnapshotLoadFSMError() throws Exception {
    final SnapshotReader reader = Mockito.mock(SnapshotReader.class);

    final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
    Mockito.when(reader.load()).thenReturn(meta);
    Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(false);
    final CountDownLatch latch = new CountDownLatch(1);
    this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {

        @Override
        public void run(final Status status) {
            assertFalse(status.isOk());
            assertEquals(-1, status.getCode());
            assertEquals("StateMachine onSnapshotLoad failed", status.getErrorMsg());
            latch.countDown();
        }

        @Override
        public SnapshotReader start() {
            return reader;
        }
    });
    latch.await();
    assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
}
 
Example #8
Source Project: sofa-jraft   Author: sofastack   File: FSMCallerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnSnapshotLoadStale() throws Exception {
    final SnapshotReader reader = Mockito.mock(SnapshotReader.class);

    final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(5).setLastIncludedTerm(1).build();
    Mockito.when(reader.load()).thenReturn(meta);

    final CountDownLatch latch = new CountDownLatch(1);
    this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {

        @Override
        public void run(final Status status) {
            assertFalse(status.isOk());
            assertEquals(RaftError.ESTALE, status.getRaftError());
            latch.countDown();
        }

        @Override
        public SnapshotReader start() {
            return reader;
        }
    });
    latch.await();
    assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
}
 
Example #9
Source Project: sofa-jraft   Author: sofastack   File: CounterStateMachine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    if (isLeader()) {
        LOG.warn("Leader is not supposed to load snapshot");
        return false;
    }
    if (reader.getFileMeta("data") == null) {
        LOG.error("Fail to find data file in {}", reader.getPath());
        return false;
    }
    final CounterSnapshotFile snapshot = new CounterSnapshotFile(reader.getPath() + File.separator + "data");
    try {
        this.value.set(snapshot.load());
        return true;
    } catch (final IOException e) {
        LOG.error("Fail to load snapshot from {}", snapshot.getPath());
        return false;
    }

}
 
Example #10
Source Project: sofa-jraft   Author: sofastack   File: AbstractKVStoreSnapshotFile.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean load(final SnapshotReader reader, final Region region) {
    final LocalFileMeta meta = (LocalFileMeta) reader.getFileMeta(SNAPSHOT_ARCHIVE);
    final String readerPath = reader.getPath();
    if (meta == null) {
        LOG.error("Can't find kv snapshot file, path={}.", readerPath);
        return false;
    }
    final String snapshotPath = Paths.get(readerPath, SNAPSHOT_DIR).toString();
    try {
        decompressSnapshot(readerPath, meta);
        doSnapshotLoad(snapshotPath, meta, region);
        final File tmp = new File(snapshotPath);
        // Delete the decompressed temporary file. If the deletion fails (although it is a small probability
        // event), it may affect the next snapshot decompression result. Therefore, the safest way is to
        // terminate the state machine immediately. Users can choose to manually delete and restart according
        // to the log information.
        if (tmp.exists()) {
            FileUtils.forceDelete(new File(snapshotPath));
        }
        return true;
    } catch (final Throwable t) {
        LOG.error("Fail to load snapshot, path={}, file list={}, {}.", readerPath, reader.listFiles(),
            StackTraceUtil.stackTrace(t));
        return false;
    }
}
 
Example #11
Source Project: sofa-jraft   Author: sofastack   File: KVStoreStateMachine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    if (isLeader()) {
        LOG.warn("Leader is not supposed to load snapshot.");
        return false;
    }
    return this.storeSnapshotFile.load(reader, this.region.copy());
}
 
Example #12
Source Project: sofa-jraft   Author: sofastack   File: MockStateMachine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    this.lastAppliedIndex.set(0);
    this.loadSnapshotTimes++;
    final String path = reader.getPath() + File.separator + "data";
    final File file = new File(path);
    if (!file.exists()) {
        return false;
    }
    try (FileInputStream fin = new FileInputStream(file); BufferedInputStream in = new BufferedInputStream(fin)) {
        this.lock.lock();
        this.logs.clear();
        try {
            while (true) {
                final byte[] bs = new byte[4];
                if (in.read(bs) == 4) {
                    final int len = Bits.getInt(bs, 0);
                    final byte[] buf = new byte[len];
                    if (in.read(buf) != len) {
                        break;
                    }
                    this.logs.add(ByteBuffer.wrap(buf));
                } else {
                    break;
                }
            }
        } finally {
            this.lock.unlock();
        }
        System.out.println("Node<" + this.address + "> loaded snapshot from " + path);
        return true;
    } catch (final IOException e) {
        e.printStackTrace();
        return false;
    }
}
 
Example #13
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotCopier.java    License: Apache License 2.0 5 votes vote down vote up
private void filter() throws IOException {
    this.writer = (LocalSnapshotWriter) this.storage.create(!this.filterBeforeCopyRemote);
    if (this.writer == null) {
        setError(RaftError.EIO, "Fail to create snapshot writer");
        return;
    }
    if (this.filterBeforeCopyRemote) {
        final SnapshotReader reader = this.storage.open();
        if (!filterBeforeCopy(this.writer, reader)) {
            LOG.warn("Fail to filter writer before copying, destroy and create a new writer.");
            this.writer.setError(-1, "Fail to filter");
            Utils.closeQuietly(this.writer);
            this.writer = (LocalSnapshotWriter) this.storage.create(true);
        }
        if (reader != null) {
            Utils.closeQuietly(reader);
        }
        if (this.writer == null) {
            setError(RaftError.EIO, "Fail to create snapshot writer");
            return;
        }
    }
    this.writer.saveMeta(this.remoteSnapshot.getMetaTable().getMeta());
    if (!this.writer.sync()) {
        LOG.error("Fail to sync snapshot writer path={}", this.writer.getPath());
        setError(RaftError.EIO, "Fail to sync snapshot writer");
    }
}
 
Example #14
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotCopierTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInterrupt() throws Exception {
    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
        .setReadPartly(true);

    //mock get metadata
    final ArgumentCaptor<RpcResponseClosure> argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
    this.copier.start();
    Thread.sleep(10);

    Utils.runInThread(new Runnable() {

        @Override
        public void run() {
            LocalSnapshotCopierTest.this.copier.cancel();
        }
    });
    this.copier.join();
    //start timer
    final SnapshotReader reader = this.copier.getReader();
    assertNull(reader);
    Assert.assertEquals(RaftError.ECANCELED.getNumber(), this.copier.getCode());
    Assert.assertEquals("Cancel the copier manually.", this.copier.getErrorMsg());
}
 
Example #15
Source Project: sofa-jraft   Author: sofastack   File: ReplicatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInstallSnapshot() {
    final Replicator r = getReplicator();
    this.id.unlock();

    final Future<Message> rpcInFly = r.getRpcInFly();
    assertNotNull(rpcInFly);
    final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
    Mockito.when(this.snapshotStorage.open()).thenReturn(reader);
    final String uri = "remote://localhost:8081/99";
    Mockito.when(reader.generateURIForCopy()).thenReturn(uri);
    final RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder() //
        .setLastIncludedIndex(11) //
        .setLastIncludedTerm(1) //
        .build();
    Mockito.when(reader.load()).thenReturn(meta);

    assertEquals(0, r.statInfo.lastLogIncluded);
    assertEquals(0, r.statInfo.lastTermIncluded);

    final RpcRequests.InstallSnapshotRequest.Builder rb = RpcRequests.InstallSnapshotRequest.newBuilder();
    rb.setTerm(this.opts.getTerm());
    rb.setGroupId(this.opts.getGroupId());
    rb.setServerId(this.opts.getServerId().toString());
    rb.setPeerId(this.opts.getPeerId().toString());
    rb.setMeta(meta);
    rb.setUri(uri);

    Mockito.when(
        this.rpcService.installSnapshot(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(rb.build()),
            Mockito.any())).thenReturn(new FutureImpl<>());

    r.installSnapshot();
    assertNotNull(r.getRpcInFly());
    assertNotSame(r.getRpcInFly(), rpcInFly);
    Assert.assertEquals(Replicator.RunningState.INSTALLING_SNAPSHOT, r.statInfo.runningState);
    assertEquals(11, r.statInfo.lastLogIncluded);
    assertEquals(1, r.statInfo.lastTermIncluded);
}
 
Example #16
Source Project: sofa-jraft   Author: sofastack   File: MockStateMachine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    this.lastAppliedIndex.set(0);
    this.loadSnapshotTimes++;
    final String path = reader.getPath() + File.separator + "data";
    final File file = new File(path);
    if (!file.exists()) {
        return false;
    }
    try (FileInputStream fin = new FileInputStream(file); BufferedInputStream in = new BufferedInputStream(fin)) {
        this.lock.lock();
        this.logs.clear();
        try {
            while (true) {
                final byte[] bs = new byte[4];
                if (in.read(bs) == 4) {
                    final int len = Bits.getInt(bs, 0);
                    final byte[] buf = new byte[len];
                    if (in.read(buf) != len) {
                        break;
                    }
                    this.logs.add(ByteBuffer.wrap(buf));
                } else {
                    break;
                }
            }
        } finally {
            this.lock.unlock();
        }
        System.out.println("Node<" + this.address + "> loaded snapshot from " + path);
        return true;
    } catch (final IOException e) {
        e.printStackTrace();
        return false;
    }
}
 
Example #17
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotCopier.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public SnapshotReader getReader() {
    return this.reader;
}
 
Example #18
Source Project: sofa-jraft   Author: sofastack   File: FSMCallerImpl.java    License: Apache License 2.0 4 votes vote down vote up
private void doSnapshotLoad(final LoadSnapshotClosure done) {
    Requires.requireNonNull(done, "LoadSnapshotClosure is null");
    final SnapshotReader reader = done.start();
    if (reader == null) {
        done.run(new Status(RaftError.EINVAL, "open SnapshotReader failed"));
        return;
    }
    final RaftOutter.SnapshotMeta meta = reader.load();
    if (meta == null) {
        done.run(new Status(RaftError.EINVAL, "SnapshotReader load meta failed"));
        if (reader.getRaftError() == RaftError.EIO) {
            final RaftException err = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT, RaftError.EIO,
                "Fail to load snapshot meta");
            setError(err);
        }
        return;
    }
    final LogId lastAppliedId = new LogId(this.lastAppliedIndex.get(), this.lastAppliedTerm);
    final LogId snapshotId = new LogId(meta.getLastIncludedIndex(), meta.getLastIncludedTerm());
    if (lastAppliedId.compareTo(snapshotId) > 0) {
        done.run(new Status(
            RaftError.ESTALE,
            "Loading a stale snapshot last_applied_index=%d last_applied_term=%d snapshot_index=%d snapshot_term=%d",
            lastAppliedId.getIndex(), lastAppliedId.getTerm(), snapshotId.getIndex(), snapshotId.getTerm()));
        return;
    }
    if (!this.fsm.onSnapshotLoad(reader)) {
        done.run(new Status(-1, "StateMachine onSnapshotLoad failed"));
        final RaftException e = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE,
            RaftError.ESTATEMACHINE, "StateMachine onSnapshotLoad failed");
        setError(e);
        return;
    }
    if (meta.getOldPeersCount() == 0) {
        // Joint stage is not supposed to be noticeable by end users.
        final Configuration conf = new Configuration();
        for (int i = 0, size = meta.getPeersCount(); i < size; i++) {
            final PeerId peer = new PeerId();
            Requires.requireTrue(peer.parse(meta.getPeers(i)), "Parse peer failed");
            conf.addPeer(peer);
        }
        this.fsm.onConfigurationCommitted(conf);
    }
    this.lastAppliedIndex.set(meta.getLastIncludedIndex());
    this.lastAppliedTerm = meta.getLastIncludedTerm();
    done.run(Status.OK());
}
 
Example #19
Source Project: sofa-jraft   Author: sofastack   File: StateMachineAdapter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    error("onSnapshotLoad", "while a snapshot is saved in " + reader.getPath());
    return false;
}
 
Example #20
Source Project: sofa-jraft   Author: sofastack   File: SnapshotExecutorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testInstallSnapshot() throws Exception {
    final RpcRequests.InstallSnapshotRequest.Builder irb = RpcRequests.InstallSnapshotRequest.newBuilder();
    irb.setGroupId("test");
    irb.setPeerId(this.addr.toString());
    irb.setServerId("localhost:8080");
    irb.setUri("remote://localhost:8080/99");
    irb.setTerm(0);
    irb.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(2));

    Mockito.when(this.raftClientService.connect(new Endpoint("localhost", 8080))).thenReturn(true);

    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
        .setReadPartly(true);

    //mock get metadata
    ArgumentCaptor<RpcResponseClosure> argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
    Utils.runInThread(new Runnable() {

        @Override
        public void run() {
            SnapshotExecutorTest.this.executor.installSnapshot(irb.build(), RpcRequests.InstallSnapshotResponse
                .newBuilder(), new RpcRequestClosure(SnapshotExecutorTest.this.asyncCtx));

        }
    });

    Thread.sleep(500);
    RpcResponseClosure<RpcRequests.GetFileResponse> closure = argument.getValue();
    final ByteBuffer metaBuf = this.table.saveToByteBufferAsRemote();
    closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(metaBuf.remaining()).setEof(true)
        .setData(ByteString.copyFrom(metaBuf)).build());

    //mock get file
    argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    rb.setFilename("testFile");
    rb.setCount(this.raftOptions.getMaxByteCountPerRpc());
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);

    closure.run(Status.OK());

    Thread.sleep(500);
    closure = argument.getValue();
    closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
        .setData(ByteString.copyFrom(new byte[100])).build());

    final ArgumentCaptor<LoadSnapshotClosure> loadSnapshotArg = ArgumentCaptor.forClass(LoadSnapshotClosure.class);
    Mockito.when(this.fSMCaller.onSnapshotLoad(loadSnapshotArg.capture())).thenReturn(true);
    closure.run(Status.OK());
    Thread.sleep(500);
    final LoadSnapshotClosure done = loadSnapshotArg.getValue();
    final SnapshotReader reader = done.start();
    assertNotNull(reader);
    assertEquals(1, reader.listFiles().size());
    assertTrue(reader.listFiles().contains("testFile"));
    done.run(Status.OK());
    this.executor.join();
    assertEquals(2, this.executor.getLastSnapshotTerm());
    assertEquals(1, this.executor.getLastSnapshotIndex());
}
 
Example #21
Source Project: sofa-jraft   Author: sofastack   File: LocalSnapshotCopierTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testStartJoinFinishOK() throws Exception {
    final FutureImpl<Message> future = new FutureImpl<>();
    final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
        .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
        .setReadPartly(true);

    //mock get metadata
    ArgumentCaptor<RpcResponseClosure> argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
    this.copier.start();
    Thread.sleep(500);
    RpcResponseClosure<RpcRequests.GetFileResponse> closure = argument.getValue();
    final ByteBuffer metaBuf = this.table.saveToByteBufferAsRemote();
    closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(metaBuf.remaining()).setEof(true)
        .setData(ByteString.copyFrom(metaBuf)).build());

    //mock get file
    argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
    rb.setFilename("testFile");
    rb.setCount(this.raftOptions.getMaxByteCountPerRpc());
    Mockito.when(
        this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
            eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);

    closure.run(Status.OK());

    Thread.sleep(500);
    closure = argument.getValue();
    closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
        .setData(ByteString.copyFrom(new byte[100])).build());
    closure.run(Status.OK());
    this.copier.join();
    final SnapshotReader reader = this.copier.getReader();
    assertSame(this.reader, reader);
    assertEquals(1, this.writer.listFiles().size());
    assertTrue(this.writer.listFiles().contains("testFile"));
}
 
Example #22
Source Project: sofa-jraft   Author: sofastack   File: NodeTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
    return false;
}
 
Example #23
Source Project: sofa-registry   Author: sofastack   File: ServiceStateMachine.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean onSnapshotLoad(SnapshotReader reader) {
    if (isLeader()) {
        LOG.warn("Leader is not supposed to load snapshot");
        return false;
    }
    List<String> failServices = new ArrayList<>();
    Map<String, Object> workers = Processor.getInstance().getWorkers();
    if (workers != null) {
        outer: for (Map.Entry<String, Object> entry : workers.entrySet()) {
            String serviceId = entry.getKey();
            Object worker = entry.getValue();
            if (worker instanceof SnapshotProcess) {
                SnapshotProcess snapshotProcess = (SnapshotProcess) worker;
                Set<String> fileNames = snapshotProcess.getSnapshotFileNames();

                for (String fileName : fileNames) {
                    if (reader.getFileMeta(fileName) == null) {
                        LOG.error("Fail to find data file {} in {}", fileName, reader.getPath());
                        failServices.add(serviceId);
                        break outer;
                    }

                    String savePath = reader.getPath() + File.separator + fileName;
                    LOG.info("Begin load snapshot path {}", savePath);
                    boolean ret = snapshotProcess.load(savePath);
                    if (!ret) {
                        LOG.error("Fail to load service:{} snapshot {}", serviceId, savePath);
                        failServices.add(serviceId);
                        break outer;
                    }
                }
            }
        }
    }

    if (!failServices.isEmpty()) {
        LOG.error("Fail to load services {} snapshot!", failServices);
        return false;
    }
    return true;
}
 
Example #24
Source Project: distkv   Author: distkv-project   File: MetaStateMachine.java    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
@Override
public boolean onSnapshotLoad(final SnapshotReader reader) {
  // TODO(kairbon): Add snapshot
  return false;
}
 
Example #25
Source Project: sofa-jraft   Author: sofastack   File: KVStoreSnapshotFile.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Load snapshot for the specified region.
 *
 * @param reader snapshot reader
 * @param region the region to load snapshot
 * @return true if load succeed
 */
boolean load(final SnapshotReader reader, final Region region);
 
Example #26
Source Project: sofa-jraft   Author: sofastack   File: StateMachine.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * User defined snapshot load function
 * get and load snapshot
 * Default: Load nothing and returns error.
 *
 * @param reader snapshot reader
 * @return true on success
 */
boolean onSnapshotLoad(final SnapshotReader reader);
 
Example #27
Source Project: sofa-jraft   Author: sofastack   File: LoadSnapshotClosure.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Start to load snapshot, returns a snapshot reader.
 *
 * @return a snapshot reader.
 */
SnapshotReader start();
 
Example #28
Source Project: sofa-jraft   Author: sofastack   File: SnapshotStorage.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Open a snapshot reader.
 */
SnapshotReader open();
 
Example #29
Source Project: sofa-jraft   Author: sofastack   File: SnapshotStorage.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Copy data from remote uri.
 *
 * @param uri  remote uri
 * @param opts copy options
 * @return a SnapshotReader instance
 */
SnapshotReader copyFrom(final String uri, final SnapshotCopierOptions opts);