Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializer#duplicate()

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializer#duplicate() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 
Example 2
Source File: StateDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 
Example 3
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 
Example 4
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 
Example 5
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 
Example 6
Source File: StateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 
Example 7
Source File: DefaultOperatorStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 
Example 8
Source File: DefaultOperatorStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 
Example 9
Source File: FunctionGroupOperator.java    From stateful-functions with Apache License 2.0 4 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
  super.initializeState(context);

  final Configuration configuration = getConfiguration();
  final StatefulFunctionsUniverse statefulFunctionsUniverse =
      statefulFunctionsUniverse(configuration);

  final TypeSerializer<Message> envelopeSerializer =
      getOperatorConfig().getTypeSerializerIn1(getContainingTask().getUserCodeClassLoader());
  final Executor checkpointLockExecutor =
      new UnderCheckpointLockExecutor(
          getContainingTask().getCheckpointLock(), () -> closedOrDisposed, getContainingTask());
  final MapStateDescriptor<Long, Message> asyncOperationStateDescriptor =
      new MapStateDescriptor<>(
          "asyncOperations", LongSerializer.INSTANCE, envelopeSerializer.duplicate());
  final ListStateDescriptor<Message> delayedMessageStateDescriptor =
      new ListStateDescriptor<>(
          FlinkStateDelayedMessagesBuffer.BUFFER_STATE_NAME, envelopeSerializer.duplicate());
  final MapState<Long, Message> asyncOperationState =
      getRuntimeContext().getMapState(asyncOperationStateDescriptor);

  Objects.requireNonNull(mailboxExecutor, "MailboxExecutor is unexpectedly NULL");

  //
  // the core logic of applying messages to functions.
  //
  this.reductions =
      Reductions.create(
          configuration,
          statefulFunctionsUniverse,
          getRuntimeContext(),
          getKeyedStateBackend(),
          new FlinkTimerServiceFactory(super.timeServiceManager),
          delayedMessagesBufferState(delayedMessageStateDescriptor),
          sideOutputs,
          output,
          MessageFactory.forType(statefulFunctionsUniverse.messageFactoryType()),
          new MailboxExecutorFacade(mailboxExecutor, "Stateful Functions Mailbox"),
          getRuntimeContext().getMetricGroup().addGroup("functions"),
          asyncOperationState,
          checkpointLockExecutor);
  //
  // expire all the pending async operations.
  //
  AsyncOperationFailureNotifier.fireExpiredAsyncOperations(
      asyncOperationStateDescriptor, reductions, asyncOperationState, getKeyedStateBackend());
}
 
Example 10
Source File: FunctionGroupOperator.java    From flink-statefun with Apache License 2.0 4 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
  super.initializeState(context);

  final StatefulFunctionsUniverse statefulFunctionsUniverse =
      statefulFunctionsUniverse(configuration);

  final TypeSerializer<Message> envelopeSerializer =
      getOperatorConfig().getTypeSerializerIn1(getContainingTask().getUserCodeClassLoader());
  final MapStateDescriptor<Long, Message> asyncOperationStateDescriptor =
      new MapStateDescriptor<>(
          "asyncOperations", LongSerializer.INSTANCE, envelopeSerializer.duplicate());
  final ListStateDescriptor<Message> delayedMessageStateDescriptor =
      new ListStateDescriptor<>(
          FlinkStateDelayedMessagesBuffer.BUFFER_STATE_NAME, envelopeSerializer.duplicate());
  final MapState<Long, Message> asyncOperationState =
      getRuntimeContext().getMapState(asyncOperationStateDescriptor);

  Objects.requireNonNull(mailboxExecutor, "MailboxExecutor is unexpectedly NULL");

  this.backPressureValve =
      new ThresholdBackPressureValve(configuration.getMaxAsyncOperationsPerTask());

  //
  // the core logic of applying messages to functions.
  //
  this.reductions =
      Reductions.create(
          backPressureValve,
          statefulFunctionsUniverse,
          getRuntimeContext(),
          getKeyedStateBackend(),
          new FlinkTimerServiceFactory(super.timeServiceManager),
          delayedMessagesBufferState(delayedMessageStateDescriptor),
          sideOutputs,
          output,
          MessageFactory.forType(statefulFunctionsUniverse.messageFactoryType()),
          new MailboxExecutorFacade(mailboxExecutor, "Stateful Functions Mailbox"),
          getRuntimeContext().getMetricGroup().addGroup("functions"),
          asyncOperationState);
  //
  // expire all the pending async operations.
  //
  AsyncOperationFailureNotifier.fireExpiredAsyncOperations(
      asyncOperationStateDescriptor, reductions, getKeyedStateBackend());
}
 
Example 11
Source File: DefaultOperatorStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 
Example 12
Source File: CollectSinkFunction.java    From flink with Apache License 2.0 4 votes vote down vote up
private ServerThread(TypeSerializer<IN> serializer) throws Exception {
	this.serializer = serializer.duplicate();
	this.serverSocket = new ServerSocket(0, 0, getBindAddress());
	this.running = true;
}