/*
* Copyright (c) 2015-2017, Christoph Engelbert (aka noctarius) and
* contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.noctarius.snowcast.impl;
import com.hazelcast.client.impl.protocol.ClientMessage;
import com.hazelcast.client.impl.protocol.codec.SnowcastRegisterChannelCodec;
import com.hazelcast.core.DistributedObject;
import com.hazelcast.instance.BuildInfo;
import com.hazelcast.instance.BuildInfoProvider;
import com.hazelcast.nio.serialization.Data;
import com.hazelcast.spi.EventPublishingService;
import com.hazelcast.spi.EventRegistration;
import com.hazelcast.spi.EventService;
import com.hazelcast.spi.InternalCompletableFuture;
import com.hazelcast.spi.InvocationBuilder;
import com.hazelcast.spi.ManagedService;
import com.hazelcast.spi.MigrationAwareService;
import com.hazelcast.spi.NodeEngine;
import com.hazelcast.spi.Operation;
import com.hazelcast.spi.OperationService;
import com.hazelcast.spi.PartitionMigrationEvent;
import com.hazelcast.spi.PartitionReplicationEvent;
import com.hazelcast.spi.RemoteService;
import com.hazelcast.spi.partition.IPartitionService;
import com.hazelcast.spi.partition.MigrationEndpoint;
import com.hazelcast.spi.serialization.SerializationService;
import com.hazelcast.util.Clock;
import com.noctarius.snowcast.SnowcastEpoch;
import com.noctarius.snowcast.SnowcastIllegalStateException;
import com.noctarius.snowcast.SnowcastSequenceState;
import com.noctarius.snowcast.SnowcastSequencer;
import com.noctarius.snowcast.SnowcastSequencerAlreadyRegisteredException;
import com.noctarius.snowcast.impl.operations.AttachLogicalNodeOperation;
import com.noctarius.snowcast.impl.operations.CreateSequencerDefinitionOperation;
import com.noctarius.snowcast.impl.operations.DestroySequencerDefinitionOperation;
import com.noctarius.snowcast.impl.operations.DetachLogicalNodeOperation;
import com.noctarius.snowcast.impl.operations.SequencerReplicationOperation;
import com.noctarius.snowcast.impl.operations.clientcodec.MessageChannel;
import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import static com.noctarius.snowcast.impl.ExceptionMessages.INTERNAL_SETUP_FAILED;
import static com.noctarius.snowcast.impl.ExceptionMessages.PARAMETER_IS_NOT_SUPPORTED;
import static com.noctarius.snowcast.impl.ExceptionMessages.SEQUENCER_ALREADY_REGISTERED;
import static com.noctarius.snowcast.impl.ExceptionMessages.UNKNOWN_HAZELCAST_VERSION;
import static com.noctarius.snowcast.impl.ExceptionUtils.exception;
import static com.noctarius.snowcast.impl.ExceptionUtils.exceptionParameters;
import static com.noctarius.snowcast.impl.ExceptionUtils.execute;
import static com.noctarius.snowcast.impl.SnowcastConstants.SERVICE_NAME;
public class NodeSequencerService
implements SequencerService, ManagedService, MigrationAwareService, RemoteService,
EventPublishingService<Object, Object> {
private static final MethodType FUTURE_GET_TYPE = MethodType.methodType(Object.class);
private static final MethodType GET_LISTENER_GET_TYPE = MethodType.methodType(Object.class);
private final ConcurrentMap<Integer, SequencerPartition> partitions;
private final ConcurrentMap<String, SequencerProvision> provisions;
private final MethodHandle getListenerMethodHandle;
private final MethodHandle futureGetMethodHandle;
private NodeEngine nodeEngine;
private EventService eventService;
private SerializationService serializationService;
public NodeSequencerService() {
this.provisions = new ConcurrentHashMap<>();
this.partitions = new ConcurrentHashMap<>();
this.getListenerMethodHandle = findEventRegistrationGetListener();
this.futureGetMethodHandle = findFutureExecutorMethod();
}
@Override
public void init(@Nonnull NodeEngine nodeEngine, @Nullable Properties properties) {
this.nodeEngine = nodeEngine;
this.eventService = nodeEngine.getEventService();
this.serializationService = nodeEngine.getSerializationService();
}
@Nonnull
@Override
public SnowcastSequencer createSequencer(@Nonnull String sequencerName, @Nonnull SnowcastEpoch epoch,
@Min(128) @Max(8192) int maxLogicalNodeCount, short backupCount) {
SequencerDefinition definition = new SequencerDefinition(sequencerName, epoch, maxLogicalNodeCount, backupCount);
Operation operation = new CreateSequencerDefinitionOperation(definition);
SequencerDefinition realDefinition = invoke(operation, sequencerName);
if (!definition.equals(realDefinition)) {
throw exception(SnowcastIllegalStateException::new, SEQUENCER_ALREADY_REGISTERED);
}
return getOrCreateSequencerProvision(realDefinition).getSequencer();
}
@Override
public void destroySequencer(@Nonnull SnowcastSequencer sequencer) {
// Remove the current provision
SequencerProvision provision = provisions.remove(sequencer.getSequencerName());
// Concurrent destroy
if (provision == null) {
return;
}
// Store destroyed state into the sequencer to prevent further id creation
provision.getSequencer().stateTransition(SnowcastSequenceState.Destroyed);
// Destroy sequencer definition in partition
Operation operation = new DestroySequencerDefinitionOperation(provision.getSequencerName());
invoke(operation, sequencer.getSequencerName());
}
@Override
public void reset() {
// No action here, however not sure if this shouldn't kill all sequencers
}
@Override
public void shutdown(boolean terminate) {
// No action here, however not sure if this shouldn't kill all sequencers
}
@Nullable
@Override
public Operation prepareReplicationOperation(@Nonnull PartitionReplicationEvent event) {
int partitionId = event.getPartitionId();
SequencerPartition partition = partitions.get(partitionId);
if (partition == null) {
return null;
}
PartitionReplication partitionReplication = partition.createPartitionReplication();
return new SequencerReplicationOperation(partitionReplication);
}
@Override
public void beforeMigration(@Nonnull PartitionMigrationEvent event) {
int partitionId = event.getPartitionId();
SequencerPartition partition = partitions.get(partitionId);
if (partition != null) {
partition.freeze();
}
}
@Override
public void commitMigration(@Nonnull PartitionMigrationEvent event) {
int partitionId = event.getPartitionId();
SequencerPartition partition = partitions.get(partitionId);
if (partition != null) {
if (event.getMigrationEndpoint() == MigrationEndpoint.SOURCE) {
partitions.remove(partitionId);
}
partition.unfreeze();
}
}
@Override
public void rollbackMigration(@Nonnull PartitionMigrationEvent event) {
int partitionId = event.getPartitionId();
SequencerPartition partition = partitions.get(partitionId);
if (partition != null) {
partition.unfreeze();
}
}
@Override
public void dispatchEvent(@Nonnull Object event, @Nonnull Object listener) {
if (listener instanceof ClientChannelHandler) {
((ClientChannelHandler) listener).onMessage(event);
}
}
@Nonnull
public SequencerDefinition registerSequencerDefinition(@Nonnull SequencerDefinition definition) {
IPartitionService partitionService = nodeEngine.getPartitionService();
int partitionId = partitionService.getPartitionId(definition.getSequencerName());
SequencerPartition partition = getSequencerPartition(partitionId);
return partition.checkOrRegisterSequencerDefinition(definition);
}
@Nullable
public SequencerDefinition destroySequencer(@Nonnull String sequencerName, boolean local) {
// Remove the current provision
SequencerProvision provision = provisions.remove(sequencerName);
SequencerDefinition definition = null;
if (provision != null) {
// Store destroyed state into the sequencer to prevent further id creation
provision.getSequencer().stateTransition(SnowcastSequenceState.Destroyed);
definition = provision.getDefinition();
}
if (local) {
// Destroy sequencer definition on partition
definition = unregisterSequencerDefinition(sequencerName);
}
return definition;
}
@Nonnull
public SequencerPartition getSequencerPartition(@Nonnegative int partitionId) {
return partitions.computeIfAbsent(partitionId, SequencerPartition::new);
}
@Nonnull
public EventRegistration registerClientChannel(@Nonnull String sequencerName, @Nonnull MessageChannel messageChannel) {
ClientChannelHandler clientChannelHandler = new ClientChannelHandler(messageChannel, serializationService);
return eventService.registerLocalListener(SnowcastConstants.SERVICE_NAME, sequencerName, clientChannelHandler);
}
public void unregisterClientChannel(@Nonnull String sequencerName, @Nonnull String registrationId) {
eventService.deregisterListener(SnowcastConstants.SERVICE_NAME, sequencerName, registrationId);
}
@Nonnull
public Collection<EventRegistration> findClientChannelRegistrations(@Nonnull String sequencerName,
@Nullable String clientUuid) {
Collection<EventRegistration> registrations = eventService
.getRegistrations(SnowcastConstants.SERVICE_NAME, sequencerName);
if (clientUuid == null) {
return registrations;
}
// Copy the registrations since the original one is not modifiable
registrations = new ArrayList<>(registrations);
Iterator<EventRegistration> iterator = registrations.iterator();
while (iterator.hasNext()) {
ClientChannelHandler channelHandler = getClientChannelHandler(iterator.next());
if (clientUuid.equals(channelHandler.messageChannel.getUuid())) {
iterator.remove();
}
}
return registrations;
}
int attachSequencer(@Nonnull SequencerDefinition definition) {
IPartitionService partitionService = nodeEngine.getPartitionService();
int partitionId = partitionService.getPartitionId(definition.getSequencerName());
AttachLogicalNodeOperation operation = new AttachLogicalNodeOperation(definition);
OperationService operationService = nodeEngine.getOperationService();
InvocationBuilder invocationBuilder = operationService.createInvocationBuilder(SERVICE_NAME, operation, partitionId);
return completableFutureGet(invocationBuilder.invoke());
}
void detachSequencer(@Nonnull SequencerDefinition definition, @Min(128) @Max(8192) int logicalNodeId) {
IPartitionService partitionService = nodeEngine.getPartitionService();
int partitionId = partitionService.getPartitionId(definition.getSequencerName());
DetachLogicalNodeOperation operation = new DetachLogicalNodeOperation(definition, logicalNodeId);
OperationService operationService = nodeEngine.getOperationService();
InvocationBuilder invocationBuilder = operationService.createInvocationBuilder(SERVICE_NAME, operation, partitionId);
completableFutureGet(invocationBuilder.invoke());
}
@Nullable
private SequencerDefinition unregisterSequencerDefinition(@Nonnull String sequencerName) {
IPartitionService partitionService = nodeEngine.getPartitionService();
int partitionId = partitionService.getPartitionId(sequencerName);
SequencerPartition partition = getSequencerPartition(partitionId);
return partition.destroySequencerDefinition(sequencerName);
}
@Nullable
private <T> T invoke(@Nonnull Operation operation, @Nonnull String sequencerName) {
return execute(() -> {
IPartitionService partitionService = nodeEngine.getPartitionService();
int partitionId = partitionService.getPartitionId(sequencerName);
OperationService operationService = nodeEngine.getOperationService();
InvocationBuilder invocationBuilder = operationService.createInvocationBuilder(SERVICE_NAME, operation, partitionId);
return completableFutureGet(invocationBuilder.invoke());
});
}
@Nonnull
private SequencerProvision getOrCreateSequencerProvision(@Nonnull SequencerDefinition definition) {
String sequencerName = definition.getSequencerName();
return provisions.computeIfAbsent(sequencerName, name -> {
NodeSequencer sequencer = new NodeSequencer(this, definition);
sequencer.attachLogicalNode();
return new SequencerProvision(definition, sequencer);
});
}
@Nonnull
@Override
public DistributedObject createDistributedObject(@Nonnull String objectName) {
return new DummyProxy(objectName);
}
@Override
public void destroyDistributedObject(@Nonnull String objectName) {
}
private <T> T completableFutureGet(InternalCompletableFuture completableFuture) {
return executeMethodHandle(futureGetMethodHandle, completableFuture);
}
private ClientChannelHandler getClientChannelHandler(EventRegistration registration) {
return executeMethodHandle(getListenerMethodHandle, registration);
}
private <T> T executeMethodHandle(MethodHandle methodHandle, Object receiver) {
try {
return (T) methodHandle.invoke(receiver);
} catch (Throwable throwable) {
//VERSION_HACK
if (throwable instanceof SnowcastSequencerAlreadyRegisteredException) {
throw (SnowcastSequencerAlreadyRegisteredException) throwable;
} else if (throwable.getCause() instanceof SnowcastSequencerAlreadyRegisteredException) {
throw (SnowcastSequencerAlreadyRegisteredException) throwable.getCause();
}
throw exception(SnowcastSequencerAlreadyRegisteredException::new, PARAMETER_IS_NOT_SUPPORTED, "completableFuture");
}
}
private MethodHandle findEventRegistrationGetListener() {
if (InternalSequencerUtils.getHazelcastVersion() != SnowcastConstants.HazelcastVersion.Unknown) {
BuildInfo buildInfo = BuildInfoProvider.getBuildInfo();
return hz37EventRegistrationGetListener(buildInfo);
}
throw exception(UNKNOWN_HAZELCAST_VERSION);
}
private MethodHandle hz37EventRegistrationGetListener(BuildInfo buildInfo) {
//ACCESSIBILITY_HACK
return execute(() -> {
Class<?> clazz = Class.forName("com.hazelcast.spi.impl.eventservice.impl.Registration");
MethodHandles.Lookup lookup = MethodHandles.lookup();
return lookup.findVirtual(clazz, "getListener", GET_LISTENER_GET_TYPE);
}, INTERNAL_SETUP_FAILED, exceptionParameters(buildInfo.getVersion()));
}
private MethodHandle findFutureExecutorMethod() {
BuildInfo buildInfo = BuildInfoProvider.getBuildInfo();
if (InternalSequencerUtils.getHazelcastVersion() == SnowcastConstants.HazelcastVersion.V_3_7) {
return getFutureExecutorMethod(buildInfo, "com.hazelcast.spi.InternalCompletableFuture", "getSafely");
} else if (InternalSequencerUtils.getHazelcastVersion() == SnowcastConstants.HazelcastVersion.V_3_8) {
return getFutureExecutorMethod(buildInfo, "java.util.concurrent.Future", "get");
}
throw exception(UNKNOWN_HAZELCAST_VERSION);
}
private MethodHandle getFutureExecutorMethod(BuildInfo buildInfo, String className, String methodName) {
//VERSION_HACK
return execute(() -> {
Class<?> clazz = Class.forName(className);
MethodHandles.Lookup lookup = MethodHandles.lookup();
return lookup.findVirtual(clazz, methodName, FUTURE_GET_TYPE);
}, INTERNAL_SETUP_FAILED, exceptionParameters(buildInfo.getVersion()));
}
private class ClientChannelHandler {
private final MessageChannel messageChannel;
private final SerializationService serializationService;
ClientChannelHandler(@Nonnull MessageChannel messageChannel, @Nonnull SerializationService serializationService) {
this.messageChannel = messageChannel;
this.serializationService = serializationService;
}
void onMessage(Object message) {
String publisherUuid = nodeEngine.getClusterService().getLocalMember().getUuid();
Data messageData = serializationService.toData(message);
//EVENT_HACK
ClientMessage eventMessage = SnowcastRegisterChannelCodec
.encodeTopicEvent(messageData, Clock.currentTimeMillis(), publisherUuid);
messageChannel.sendClientMessage(eventMessage);
}
}
}
