io.atomix.protocols.raft.storage.RaftStorage Java Examples

/**
 * Creates a Raft server.
 */
private RaftServer createServer(RaftMember member) {
  RaftServerProtocol protocol;
  if (USE_NETTY) {
    Address address = Address.from(++port);
    MessagingService messagingManager = new NettyMessagingService("test", address, new MessagingConfig()).start().join();
    messagingServices.add(messagingManager);
    addressMap.put(member.memberId(), address);
    protocol = new RaftServerMessagingProtocol(messagingManager, PROTOCOL_SERIALIZER, addressMap::get);
  } else {
    protocol = protocolFactory.newServerProtocol(member.memberId());
  }

  RaftServer.Builder builder = RaftServer.builder(member.memberId())
      .withProtocol(protocol)
      .withStorage(RaftStorage.builder()
          .withStorageLevel(StorageLevel.DISK)
          .withDirectory(new File(String.format("target/fuzz-logs/%s", member.memberId())))
          .withNamespace(STORAGE_NAMESPACE)
          .withMaxSegmentSize(1024 * 1024)
          .build());

  RaftServer server = builder.build();
  servers.add(server);
  return server;
}
 

public MetaStore(RaftStorage storage, Serializer serializer) {
  this.serializer = checkNotNull(serializer, "serializer cannot be null");

  if (!(storage.directory().isDirectory() || storage.directory().mkdirs())) {
    throw new IllegalArgumentException(String.format("Can't create storage directory [%s].", storage.directory()));
  }

  // Note that for raft safety, irrespective of the storage level, <term, vote> metadata is always persisted on disk.
  File metaFile = new File(storage.directory(), String.format("%s.meta", storage.prefix()));
  metadataBuffer = FileBuffer.allocate(metaFile, 12);

  if (storage.storageLevel() == StorageLevel.MEMORY) {
    configurationBuffer = HeapBuffer.allocate(32);
  } else {
    File confFile = new File(storage.directory(), String.format("%s.conf", storage.prefix()));
    configurationBuffer = FileBuffer.allocate(confFile, 32);
  }
}
 

/**
 * Returns a new snapshot store.
 */
protected SnapshotStore createSnapshotStore() {
  RaftStorage storage = RaftStorage.builder()
      .withPrefix("test")
      .withDirectory(new File(String.format("target/test-logs/%s", testId)))
      .withStorageLevel(StorageLevel.DISK)
      .build();
  return new SnapshotStore(storage);
}
 

/**
 * Returns a new snapshot store.
 */
protected SnapshotStore createSnapshotStore() {
  RaftStorage storage = RaftStorage.builder()
      .withPrefix("test")
      .withStorageLevel(StorageLevel.MEMORY)
      .build();
  return new SnapshotStore(storage);
}
 

@Before
public void setupContext() throws IOException {
  deleteStorage();

  RaftStorage storage = RaftStorage.builder()
      .withPrefix("test")
      .withDirectory(PATH.toFile())
      .withNamespace(NAMESPACE)
      .build();
  PrimitiveTypeRegistry registry = new PrimitiveTypeRegistry() {
    @Override
    public Collection<PrimitiveType> getPrimitiveTypes() {
      return Collections.singleton(new TestType());
    }

    @Override
    public PrimitiveType getPrimitiveType(String typeName) {
      return new TestType();
    }
  };
  raft = new RaftContext(
      "test",
      MemberId.from("test-1"),
      mock(ClusterMembershipService.class),
      mock(RaftServerProtocol.class),
      storage,
      registry,
      ThreadModel.SHARED_THREAD_POOL.factory("raft-server-test-%d", 1, LoggerFactory.getLogger(RaftServer.class)),
      true);

  snapshotTaken = new AtomicBoolean();
  snapshotInstalled = new AtomicBoolean();
}
 

private RaftServer recreateServerWithDataLoss(RaftMember leader, RaftMember member, RaftServer server, RaftStorage storage) throws TimeoutException {
  server.shutdown().thenRun(this::resume);
  await(30000);
  deleteStorage(storage);

  final RaftServer newServer = createServer(member.memberId(), b -> b.withStorage(storage));
  newServer.bootstrap(leader.memberId()).thenRun(this::resume);
  await(30000);
  return newServer;
}
 

private RaftServer buildServer() {
  return RaftServer.builder(localMemberId)
      .withName(partition.name())
      .withMembershipService(membershipService)
      .withProtocol(new RaftServerCommunicator(
          partition.name(),
          Serializer.using(RaftNamespaces.RAFT_PROTOCOL),
          clusterCommunicator))
      .withPrimitiveTypes(primitiveTypes)
      .withElectionTimeout(config.getElectionTimeout())
      .withHeartbeatInterval(config.getHeartbeatInterval())
      .withSessionTimeout(config.getDefaultSessionTimeout())
      .withStorage(RaftStorage.builder()
          .withPrefix(partition.name())
          .withDirectory(partition.dataDirectory())
          .withStorageLevel(config.getStorageConfig().getLevel())
          .withMaxSegmentSize((int) config.getStorageConfig().getSegmentSize().bytes())
          .withMaxEntrySize((int) config.getStorageConfig().getMaxEntrySize().bytes())
          .withFlushOnCommit(config.getStorageConfig().isFlushOnCommit())
          .withDynamicCompaction(config.getCompactionConfig().isDynamic())
          .withFreeDiskBuffer(config.getCompactionConfig().getFreeDiskBuffer())
          .withFreeMemoryBuffer(config.getCompactionConfig().getFreeMemoryBuffer())
          .withNamespace(RaftNamespaces.RAFT_STORAGE)
          .build())
      .withThreadContextFactory(threadContextFactory)
      .build();
}
 

private RaftStorage createStorage(MemberId memberId, Function<RaftStorage.Builder, RaftStorage.Builder> configurator) {
  final RaftStorage.Builder defaults =
      RaftStorage.builder()
          .withStorageLevel(StorageLevel.DISK)
          .withDirectory(new File(String.format("target/test-logs/%s", memberId)))
          .withMaxEntriesPerSegment(10)
          .withMaxSegmentSize(1024 * 10)
          .withNamespace(NAMESPACE);
  return configurator.apply(defaults).build();
}
 

private void deleteStorage(RaftStorage storage) {
  storage.deleteSnapshotStore();
  storage.deleteLog();
  storage.deleteMetaStore();
}
 

/**
 * Creates a Raft server.
 */
private RaftServer createServer(Member member, List<Node> members) {
  RaftServerProtocol protocol;
  ManagedMessagingService messagingService;
  if (USE_NETTY) {
    messagingService = (ManagedMessagingService) new NettyMessagingService("test", member.address(), new MessagingConfig())
        .start()
        .join();
    messagingServices.add(messagingService);
    protocol = new RaftServerMessagingProtocol(messagingService, PROTOCOL_SERIALIZER, addressMap::get);
  } else {
    protocol = protocolFactory.newServerProtocol(member.id());
  }

  BootstrapService bootstrapService = new BootstrapService() {
    @Override
    public MessagingService getMessagingService() {
      return messagingService;
    }

    @Override
    public UnicastService getUnicastService() {
      return new UnicastServiceAdapter();
    }

    @Override
    public BroadcastService getBroadcastService() {
      return new BroadcastServiceAdapter();
    }
  };

  RaftServer.Builder builder = RaftServer.builder(member.id())
      .withProtocol(protocol)
      .withThreadModel(ThreadModel.SHARED_THREAD_POOL)
      .withMembershipService(new DefaultClusterMembershipService(
          member,
          Version.from("1.0.0"),
          new DefaultNodeDiscoveryService(bootstrapService, member, new BootstrapDiscoveryProvider(members)),
          bootstrapService,
          new HeartbeatMembershipProtocol(new HeartbeatMembershipProtocolConfig())))
      .withStorage(RaftStorage.builder()
          .withStorageLevel(StorageLevel.DISK)
          .withDirectory(new File(String.format("target/perf-logs/%s", member.id())))
          .withNamespace(STORAGE_NAMESPACE)
          .withMaxSegmentSize(1024 * 1024 * 64)
          .withDynamicCompaction()
          .withFlushOnCommit(false)
          .build());

  RaftServer server = builder.build();
  servers.add(server);
  return server;
}
 

private RaftStorage createStorage(MemberId memberId) {
  return createStorage(memberId, Function.identity());
}
 

private RaftServer createServer(Map.Entry<MemberId, RaftStorage> entry) {
  return createServer(entry.getKey(), b -> b.withStorage(entry.getValue()));
}
 

public SnapshotStore(RaftStorage storage) {
  this.storage = checkNotNull(storage, "storage cannot be null");
  open();
}
 

@Override
public RaftServer build() {
  Logger log = ContextualLoggerFactory.getLogger(RaftServer.class, LoggerContext.builder(RaftServer.class)
      .addValue(name)
      .build());

  if (primitiveTypes == null) {
    primitiveTypes = new ClasspathScanningPrimitiveTypeRegistry(Thread.currentThread().getContextClassLoader());
  }
  if (primitiveTypes.getPrimitiveTypes().isEmpty()) {
    throw new IllegalStateException("No primitive services registered");
  }

  // If the server name is null, set it to the member ID.
  if (name == null) {
    name = localMemberId.id();
  }

  // If the storage is not configured, create a new Storage instance with the configured serializer.
  if (storage == null) {
    storage = RaftStorage.builder().build();
  }

  // If a ThreadContextFactory was not provided, create one and ensure it's closed when the server is stopped.
  boolean closeOnStop;
  ThreadContextFactory threadContextFactory;
  if (this.threadContextFactory == null) {
    threadContextFactory = threadModel.factory("raft-server-" + name + "-%d", threadPoolSize, log);
    closeOnStop = true;
  } else {
    threadContextFactory = this.threadContextFactory;
    closeOnStop = false;
  }

  RaftContext raft = new RaftContext(
      name,
      localMemberId,
      membershipService,
      protocol,
      storage,
      primitiveTypes,
      threadContextFactory,
      closeOnStop);
  raft.setElectionTimeout(electionTimeout);
  raft.setHeartbeatInterval(heartbeatInterval);
  raft.setSessionTimeout(sessionTimeout);

  return new DefaultRaftServer(raft);
}
 

@SuppressWarnings("unchecked")
public RaftContext(
    String name,
    MemberId localMemberId,
    ClusterMembershipService membershipService,
    RaftServerProtocol protocol,
    RaftStorage storage,
    PrimitiveTypeRegistry primitiveTypes,
    ThreadContextFactory threadContextFactory,
    boolean closeOnStop) {
  this.name = checkNotNull(name, "name cannot be null");
  this.membershipService = checkNotNull(membershipService, "membershipService cannot be null");
  this.protocol = checkNotNull(protocol, "protocol cannot be null");
  this.storage = checkNotNull(storage, "storage cannot be null");
  this.primitiveTypes = checkNotNull(primitiveTypes, "registry cannot be null");
  this.log = ContextualLoggerFactory.getLogger(getClass(), LoggerContext.builder(RaftServer.class)
      .addValue(name)
      .build());

  // Lock the storage directory.
  if (!storage.lock(localMemberId.id())) {
    throw new StorageException("Failed to acquire storage lock; ensure each Raft server is configured with a distinct storage directory");
  }

  String baseThreadName = String.format("raft-server-%s", name);
  this.threadContext = new SingleThreadContext(namedThreads(baseThreadName, log));
  this.loadContext = new SingleThreadContext(namedThreads(baseThreadName + "-load", log));
  this.stateContext = new SingleThreadContext(namedThreads(baseThreadName + "-state", log));

  this.threadContextFactory = checkNotNull(threadContextFactory, "threadContextFactory cannot be null");
  this.closeOnStop = closeOnStop;

  this.loadMonitor = new LoadMonitor(LOAD_WINDOW_SIZE, HIGH_LOAD_THRESHOLD, loadContext);

  // Open the metadata store.
  this.meta = storage.openMetaStore();

  // Load the current term and last vote from disk.
  this.term = meta.loadTerm();
  this.lastVotedFor = meta.loadVote();

  // Construct the core log, reader, writer, and compactor.
  this.raftLog = storage.openLog();
  this.logWriter = raftLog.writer();
  this.logReader = raftLog.openReader(1, RaftLogReader.Mode.ALL);

  // Open the snapshot store.
  this.snapshotStore = storage.openSnapshotStore();

  // Create a new internal server state machine.
  this.stateMachine = new RaftServiceManager(this, stateContext, threadContextFactory);

  this.cluster = new RaftClusterContext(localMemberId, this);

  // Register protocol listeners.
  registerHandlers(protocol);
}
 

/**
 * Sets the storage module.
 *
 * @param storage The storage module.
 * @return The Raft server builder.
 * @throws NullPointerException if {@code storage} is null
 */
public Builder withStorage(RaftStorage storage) {
  this.storage = checkNotNull(storage, "storage cannot be null");
  return this;
}
 

/**
 * Returns the server storage.
 *
 * @return The server storage.
 */
public RaftStorage getStorage() {
  return storage;
}