Java Code Examples for org.apache.flink.runtime.state.OperatorStateHandle#StateMetaInfo

The following examples show how to use org.apache.flink.runtime.state.OperatorStateHandle#StateMetaInfo . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionSplitDistributeStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(1);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[30]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{0, 15}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[40]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 2

Source File: StateAssignmentOperationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 3

Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0

6 votes

void serializeOperatorStateHandle(OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}

Example 4

Source File: StateHandleDummyUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the given {@link OperatorStreamStateHandle}.
 */
public static OperatorStateHandle deepDummyCopy(OperatorStateHandle original) {

	if (original == null) {
		return null;
	}

	ByteStreamStateHandle stateHandleCopy = cloneByteStreamStateHandle((ByteStreamStateHandle) original.getDelegateStateHandle());
	Map<String, OperatorStateHandle.StateMetaInfo> offsets = original.getStateNameToPartitionOffsets();
	Map<String, OperatorStateHandle.StateMetaInfo> offsetsCopy = new HashMap<>(offsets.size());

	for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : offsets.entrySet()) {
		OperatorStateHandle.StateMetaInfo metaInfo = entry.getValue();
		OperatorStateHandle.StateMetaInfo metaInfoCopy =
			new OperatorStateHandle.StateMetaInfo(metaInfo.getOffsets(), metaInfo.getDistributionMode());
		offsetsCopy.put(String.valueOf(entry.getKey()), metaInfoCopy);
	}
	return new OperatorStreamStateHandle(offsetsCopy, stateHandleCopy);
}

Example 5

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example 6

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}

Example 7

Source File: SavepointV2Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeOperatorStateHandle(
	OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}

Example 8

Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public boolean hasNext() {

	if (null != offsets && offPos < offsets.length) {

		return true;
	}

	closeCurrentStream();

	while (stateHandleIterator.hasNext()) {
		currentStateHandle = stateHandleIterator.next();
		OperatorStateHandle.StateMetaInfo metaInfo =
			currentStateHandle.getStateNameToPartitionOffsets().get(stateName);

		if (null != metaInfo) {
			long[] metaOffsets = metaInfo.getOffsets();
			if (null != metaOffsets && metaOffsets.length > 0) {
				this.offsets = metaOffsets;
				this.offPos = 0;

				if (closableRegistry.unregisterCloseable(currentStream)) {
					IOUtils.closeQuietly(currentStream);
					currentStream = null;
				}

				return true;
			}
		}
	}

	return false;
}

Example 9

Source File: RoundRobinOperatorStateRepartitioner.java From flink with Apache License 2.0

5 votes

/**
 * Collect union states from given parallelSubtaskStates.
 */
private Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> collectUnionStates(
	List<List<OperatorStateHandle>> parallelSubtaskStates) {

	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> unionStates =
		new HashMap<>(parallelSubtaskStates.size());

	for (List<OperatorStateHandle> subTaskState : parallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : subTaskState) {
			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			partitionOffsetEntries.stream()
				.filter(entry -> entry.getValue().getDistributionMode().equals(OperatorStateHandle.Mode.UNION))
				.forEach(entry -> {
					List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
						unionStates.computeIfAbsent(entry.getKey(), k -> new ArrayList<>(parallelSubtaskStates.size() * partitionOffsetEntries.size()));

					stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), entry.getValue()));
				});
		}
	}

	return unionStates;
}

Example 10

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

5 votes

@Override
public boolean hasNext() {

	if (null != offsets && offPos < offsets.length) {

		return true;
	}

	closeCurrentStream();

	while (stateHandleIterator.hasNext()) {
		currentStateHandle = stateHandleIterator.next();
		OperatorStateHandle.StateMetaInfo metaInfo =
			currentStateHandle.getStateNameToPartitionOffsets().get(stateName);

		if (null != metaInfo) {
			long[] metaOffsets = metaInfo.getOffsets();
			if (null != metaOffsets && metaOffsets.length > 0) {
				this.offsets = metaOffsets;
				this.offPos = 0;

				if (closableRegistry.unregisterCloseable(currentStream)) {
					IOUtils.closeQuietly(currentStream);
					currentStream = null;
				}

				return true;
			}
		}
	}

	return false;
}

Example 11

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

5 votes

private static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
	try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
			for (long offset : entry.getValue().getOffsets()) {
				in.seek(offset);
				Integer state = InstantiationUtil.
						deserializeObject(in, Thread.currentThread().getContextClassLoader());
				resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
			}
		}
	}
}

Example 12

Source File: RoundRobinOperatorStateRepartitioner.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Repartition all named states.
 */
private List<Map<StreamStateHandle, OperatorStateHandle>> repartition(
		GroupByStateNameResults nameToStateByMode,
		int newParallelism) {

	// We will use this to merge w.r.t. StreamStateHandles for each parallel subtask inside the maps
	List<Map<StreamStateHandle, OperatorStateHandle>> mergeMapList = new ArrayList<>(newParallelism);

	// Initialize
	for (int i = 0; i < newParallelism; ++i) {
		mergeMapList.add(new HashMap<>());
	}

	// Start with the state handles we distribute round robin by splitting by offsets
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToDistributeState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);

	repartitionSplitState(nameToDistributeState, newParallelism, mergeMapList);

	// Now we also add the state handles marked for union to all parallel instances
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToUnionState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.UNION);

	repartitionUnionState(nameToUnionState, mergeMapList);

	// Now we also add the state handles marked for uniform broadcast to all parallel instances
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToBroadcastState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.BROADCAST);

	repartitionBroadcastState(nameToBroadcastState, mergeMapList);

	return mergeMapList;
}

Example 13

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

5 votes

static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
	try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
			for (long offset : entry.getValue().getOffsets()) {
				in.seek(offset);
				Integer state = InstantiationUtil.
					deserializeObject(in, Thread.currentThread().getContextClassLoader());
				resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
			}
		}
	}
}

Example 14

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

5 votes

/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{52, 63}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{67, 74, 75}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{77, 88, 92}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{101, 123, 127}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
	metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{20, 27, 28}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{30, 44, 48}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{57, 79, 83}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	// rescale up case, parallelism 2 --> 3
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);

	// rescale down case, parallelism 2 --> 1
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);

	// not rescale
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}

Example 15

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeOperatorStateHandle(
	OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}

Example 16

Source File: RoundRobinOperatorStateRepartitioner.java From flink with Apache License 2.0

4 votes

/**
 * Group by the different named states.
 */
@SuppressWarnings("unchecked, rawtype")
private GroupByStateNameResults groupByStateMode(List<List<OperatorStateHandle>> previousParallelSubtaskStates) {

	//Reorganize: group by (State Name -> StreamStateHandle + StateMetaInfo)
	EnumMap<OperatorStateHandle.Mode,
			Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> nameToStateByMode =
			new EnumMap<>(OperatorStateHandle.Mode.class);

	for (OperatorStateHandle.Mode mode : OperatorStateHandle.Mode.values()) {

		nameToStateByMode.put(
				mode,
				new HashMap<>());
	}

	for (List<OperatorStateHandle> previousParallelSubtaskState : previousParallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : previousParallelSubtaskState) {

			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : partitionOffsetEntries) {
				OperatorStateHandle.StateMetaInfo metaInfo = e.getValue();

				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToState =
					nameToStateByMode.get(metaInfo.getDistributionMode());

				List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
					nameToState.computeIfAbsent(
						e.getKey(),
						k -> new ArrayList<>(previousParallelSubtaskStates.size() * partitionOffsetEntries.size()));

				stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), e.getValue()));
			}
		}
	}

	return new GroupByStateNameResults(nameToStateByMode);
}

Example 17

Source File: CheckpointTestUtils.java From flink with Apache License 2.0

4 votes

/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}

Example 18

Source File: RoundRobinOperatorStateRepartitioner.java From flink with Apache License 2.0

4 votes

GroupByStateNameResults(
		EnumMap<OperatorStateHandle.Mode,
				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> byMode) {
	this.byMode = Preconditions.checkNotNull(byMode);
}

Example 19

Source File: RoundRobinOperatorStateRepartitioner.java From flink with Apache License 2.0

4 votes

@Override
public List<List<OperatorStateHandle>> repartitionState(
		List<List<OperatorStateHandle>> previousParallelSubtaskStates,
		int oldParallelism,
		int newParallelism) {

	Preconditions.checkNotNull(previousParallelSubtaskStates);
	Preconditions.checkArgument(newParallelism > 0);
	Preconditions.checkArgument(previousParallelSubtaskStates.size() == oldParallelism,
		"This method still depends on the order of the new and old operators");

	// Assemble result from all merge maps
	List<List<OperatorStateHandle>> result = new ArrayList<>(newParallelism);

	List<Map<StreamStateHandle, OperatorStateHandle>> mergeMapList;

	// We only round-robin repartition UNION state if new parallelism equals to the old one.
	if (newParallelism == oldParallelism) {
		Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> unionStates = collectUnionStates(previousParallelSubtaskStates);

		if (unionStates.isEmpty()) {
			return previousParallelSubtaskStates;
		}

		// Initialize
		mergeMapList = initMergeMapList(previousParallelSubtaskStates);

		repartitionUnionState(unionStates, mergeMapList);
	} else {

		// Reorganize: group by (State Name -> StreamStateHandle + Offsets)
		GroupByStateNameResults nameToStateByMode = groupByStateMode(previousParallelSubtaskStates);

		if (OPTIMIZE_MEMORY_USE) {
			previousParallelSubtaskStates.clear(); // free for GC at to cost that old handles are no longer available
		}

		// Do the actual repartitioning for all named states
		mergeMapList =
			repartition(nameToStateByMode, newParallelism);
	}

	for (int i = 0; i < mergeMapList.size(); ++i) {
		result.add(i, new ArrayList<>(mergeMapList.get(i).values()));
	}

	return result;
}

Example 20

Source File: RoundRobinOperatorStateRepartitioner.java From flink with Apache License 2.0

4 votes

/**
 * Group by the different named states.
 */
@SuppressWarnings("unchecked, rawtype")
private GroupByStateNameResults groupByStateMode(List<List<OperatorStateHandle>> previousParallelSubtaskStates) {

	//Reorganize: group by (State Name -> StreamStateHandle + StateMetaInfo)
	EnumMap<OperatorStateHandle.Mode,
			Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> nameToStateByMode =
			new EnumMap<>(OperatorStateHandle.Mode.class);

	for (OperatorStateHandle.Mode mode : OperatorStateHandle.Mode.values()) {

		nameToStateByMode.put(
				mode,
				new HashMap<>());
	}

	for (List<OperatorStateHandle> previousParallelSubtaskState : previousParallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : previousParallelSubtaskState) {

			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : partitionOffsetEntries) {
				OperatorStateHandle.StateMetaInfo metaInfo = e.getValue();

				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToState =
					nameToStateByMode.get(metaInfo.getDistributionMode());

				List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
					nameToState.computeIfAbsent(
						e.getKey(),
						k -> new ArrayList<>(previousParallelSubtaskStates.size() * partitionOffsetEntries.size()));

				stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), e.getValue()));
			}
		}
	}

	return new GroupByStateNameResults(nameToStateByMode);
}