org.apache.flink.runtime.state.OperatorStreamStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.OperatorStreamStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #2

Source File: JobMasterTest.java From flink with Apache License 2.0

6 votes

private Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
	Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);

	for (OperatorID operatorId : operatorIds) {
		final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
		final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
			new OperatorStreamStateHandle(
				Collections.emptyMap(),
				new ByteStreamStateHandle("foobar", new byte[0])),
			null,
			null,
			null);
		operatorState.putState(0, subtaskState);
		operatorStates.add(operatorState);
	}

	return operatorStates;
}

Example #3

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionBroadcastState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(2);
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[60]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #4

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #5

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionSplitDistributeStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(1);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[30]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{0, 15}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[40]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #6

Source File: PrioritizedOperatorSubtaskStateTest.java From flink with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the first state object in the given collection, or null if the collection is empy.
 */
private <T extends StateObject> T deepCopyFirstElement(StateObjectCollection<T> original) {
	if (original.isEmpty()) {
		return null;
	}

	T stateObject = original.iterator().next();
	StateObject result;
	if (stateObject instanceof OperatorStreamStateHandle) {
		result = deepDummyCopy((OperatorStateHandle) stateObject);
	} else if (stateObject instanceof KeyedStateHandle) {
		result = deepDummyCopy((KeyedStateHandle) stateObject);
	} else {
		throw new IllegalStateException();
	}
	return (T) result;
}

Example #7

Source File: StateHandleDummyUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the given {@link OperatorStreamStateHandle}.
 */
public static OperatorStateHandle deepDummyCopy(OperatorStateHandle original) {

	if (original == null) {
		return null;
	}

	ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) original.getDelegateStateHandle();
	ByteStreamStateHandle stateHandleCopy = new ByteStreamStateHandle(
		String.valueOf(stateHandle.getHandleName()),
		stateHandle.getData().clone());
	Map<String, OperatorStateHandle.StateMetaInfo> offsets = original.getStateNameToPartitionOffsets();
	Map<String, OperatorStateHandle.StateMetaInfo> offsetsCopy = new HashMap<>(offsets.size());

	for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : offsets.entrySet()) {
		OperatorStateHandle.StateMetaInfo metaInfo = entry.getValue();
		OperatorStateHandle.StateMetaInfo metaInfoCopy =
			new OperatorStateHandle.StateMetaInfo(metaInfo.getOffsets(), metaInfo.getDistributionMode());
		offsetsCopy.put(String.valueOf(entry.getKey()), metaInfoCopy);
	}
	return new OperatorStreamStateHandle(offsetsCopy, stateHandleCopy);
}

Example #8

Source File: StateHandleDummyUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates a new test {@link OperatorStreamStateHandle} with a given number of randomly created named states.
 */
public static OperatorStateHandle createNewOperatorStateHandle(int numNamedStates, Random random) {
	Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetaData = new HashMap<>(numNamedStates);
	byte[] streamData = new byte[numNamedStates * 4];
	random.nextBytes(streamData);
	long off = 0;
	for (int i = 0; i < numNamedStates; ++i) {
		long[] offsets = new long[4];
		for (int o = 0; o < offsets.length; ++o) {
			offsets[o] = off++;
		}
		OperatorStateHandle.StateMetaInfo metaInfo =
			new OperatorStateHandle.StateMetaInfo(offsets, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
		operatorStateMetaData.put(String.valueOf(UUID.randomUUID()), metaInfo);
	}
	ByteStreamStateHandle byteStreamStateHandle =
		new ByteStreamStateHandle(String.valueOf(UUID.randomUUID()), streamData);
	return new OperatorStreamStateHandle(operatorStateMetaData, byteStreamStateHandle);
}

Example #9

Source File: StateHandleDummyUtil.java From flink with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the given {@link OperatorStreamStateHandle}.
 */
public static OperatorStateHandle deepDummyCopy(OperatorStateHandle original) {

	if (original == null) {
		return null;
	}

	ByteStreamStateHandle stateHandleCopy = cloneByteStreamStateHandle((ByteStreamStateHandle) original.getDelegateStateHandle());
	Map<String, OperatorStateHandle.StateMetaInfo> offsets = original.getStateNameToPartitionOffsets();
	Map<String, OperatorStateHandle.StateMetaInfo> offsetsCopy = new HashMap<>(offsets.size());

	for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : offsets.entrySet()) {
		OperatorStateHandle.StateMetaInfo metaInfo = entry.getValue();
		OperatorStateHandle.StateMetaInfo metaInfoCopy =
			new OperatorStateHandle.StateMetaInfo(metaInfo.getOffsets(), metaInfo.getDistributionMode());
		offsetsCopy.put(String.valueOf(entry.getKey()), metaInfoCopy);
	}
	return new OperatorStreamStateHandle(offsetsCopy, stateHandleCopy);
}

Example #10

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionSplitDistributeStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(1);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[30]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{0, 15}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[40]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #11

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRepartitionBroadcastState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(2);
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[60]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #12

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
	Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);

	for (OperatorID operatorId : operatorIds) {
		final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
		final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
			new OperatorStreamStateHandle(
				Collections.emptyMap(),
				new ByteStreamStateHandle("foobar", new byte[0])),
			null,
			null,
			null);
		operatorState.putState(0, subtaskState);
		operatorStates.add(operatorState);
	}

	return operatorStates;
}

Example #13

Source File: CheckpointMetadataLoadingTest.java From flink with Apache License 2.0

6 votes

private static CompletedCheckpointStorageLocation createSavepointWithOperatorSubtaskState(
		final long checkpointId,
		final OperatorID operatorId,
		final int parallelism) throws IOException {

	final Random rnd = new Random();

	final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
		new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("testHandler", new byte[0])),
		null,
		null,
		null,
		singleton(createNewInputChannelStateHandle(10, rnd)),
		singleton(createNewResultSubpartitionStateHandle(10, rnd)));

	final OperatorState state = new OperatorState(operatorId, parallelism, parallelism);
	state.putState(0, subtaskState);

	return createSavepointWithOperatorState(checkpointId, state);
}

Example #14

Source File: StateAssignmentOperationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testRepartitionBroadcastState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[60]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(2);
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.BROADCAST));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[60]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #15

Source File: StateAssignmentOperationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #16

Source File: StateAssignmentOperationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testRepartitionSplitDistributeStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(1);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[30]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{0, 15}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[40]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}

Example #17

Source File: PrioritizedOperatorSubtaskStateTest.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the first state object in the given collection, or null if the collection is empy.
 */
private <T extends StateObject> T deepCopyFirstElement(StateObjectCollection<T> original) {
	if (original.isEmpty()) {
		return null;
	}

	T stateObject = original.iterator().next();
	StateObject result;
	if (stateObject instanceof OperatorStreamStateHandle) {
		result = deepDummyCopy((OperatorStateHandle) stateObject);
	} else if (stateObject instanceof KeyedStateHandle) {
		result = deepDummyCopy((KeyedStateHandle) stateObject);
	} else {
		throw new IllegalStateException();
	}
	return (T) result;
}

Example #18

Source File: StateHandleDummyUtil.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Creates a deep copy of the given {@link OperatorStreamStateHandle}.
 */
public static OperatorStateHandle deepDummyCopy(OperatorStateHandle original) {

	if (original == null) {
		return null;
	}

	ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) original.getDelegateStateHandle();
	ByteStreamStateHandle stateHandleCopy = new ByteStreamStateHandle(
		String.valueOf(stateHandle.getHandleName()),
		stateHandle.getData().clone());
	Map<String, OperatorStateHandle.StateMetaInfo> offsets = original.getStateNameToPartitionOffsets();
	Map<String, OperatorStateHandle.StateMetaInfo> offsetsCopy = new HashMap<>(offsets.size());

	for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : offsets.entrySet()) {
		OperatorStateHandle.StateMetaInfo metaInfo = entry.getValue();
		OperatorStateHandle.StateMetaInfo metaInfoCopy =
			new OperatorStateHandle.StateMetaInfo(metaInfo.getOffsets(), metaInfo.getDistributionMode());
		offsetsCopy.put(String.valueOf(entry.getKey()), metaInfoCopy);
	}
	return new OperatorStreamStateHandle(offsetsCopy, stateHandleCopy);
}

Example #19

Source File: StateHandleDummyUtil.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Creates a new test {@link OperatorStreamStateHandle} with a given number of randomly created named states.
 */
public static OperatorStateHandle createNewOperatorStateHandle(int numNamedStates, Random random) {
	Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetaData = new HashMap<>(numNamedStates);
	byte[] streamData = new byte[numNamedStates * 4];
	random.nextBytes(streamData);
	long off = 0;
	for (int i = 0; i < numNamedStates; ++i) {
		long[] offsets = new long[4];
		for (int o = 0; o < offsets.length; ++o) {
			offsets[o] = off++;
		}
		OperatorStateHandle.StateMetaInfo metaInfo =
			new OperatorStateHandle.StateMetaInfo(offsets, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
		operatorStateMetaData.put(String.valueOf(UUID.randomUUID()), metaInfo);
	}
	ByteStreamStateHandle byteStreamStateHandle =
		new ByteStreamStateHandle(String.valueOf(UUID.randomUUID()), streamData);
	return new OperatorStreamStateHandle(operatorStateMetaData, byteStreamStateHandle);
}

Example #20

Source File: JobMasterTest.java From flink with Apache License 2.0

6 votes

private Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
	Random random = new Random();
	Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);

	for (OperatorID operatorId : operatorIds) {
		final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
		final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
			new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("foobar", new byte[0])),
			null,
			null,
			null,
			singleton(createNewInputChannelStateHandle(10, random)),
			singleton(createNewResultSubpartitionStateHandle(10, random)));
		operatorState.putState(0, subtaskState);
		operatorStates.add(operatorState);
	}

	return operatorStates;
}

Example #21

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

5 votes

/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{52, 63}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{67, 74, 75}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{77, 88, 92}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{101, 123, 127}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
	metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{20, 27, 28}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{30, 44, 48}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{57, 79, 83}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	// rescale up case, parallelism 2 --> 3
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);

	// rescale down case, parallelism 2 --> 1
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);

	// not rescale
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}

Example #22

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
public static OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}

Example #23

Source File: PrioritizedOperatorSubtaskStateTest.java From flink with Apache License 2.0

5 votes

private static StateObject deepCopy(StateObject stateObject) {
	if (stateObject instanceof OperatorStreamStateHandle) {
		return deepDummyCopy((OperatorStateHandle) stateObject);
	} else if (stateObject instanceof KeyedStateHandle) {
		return deepDummyCopy((KeyedStateHandle) stateObject);
	} else if (stateObject instanceof InputChannelStateHandle) {
		return deepDummyCopy((InputChannelStateHandle) stateObject);
	} else if (stateObject instanceof ResultSubpartitionStateHandle) {
		return deepDummyCopy((ResultSubpartitionStateHandle) stateObject);
	} else {
		throw new IllegalStateException();
	}
}

Example #24

Source File: StateHandleDummyUtil.java From flink with Apache License 2.0

5 votes

/**
 * Creates a new test {@link OperatorStreamStateHandle} with a given number of randomly created named states.
 */
public static OperatorStateHandle createNewOperatorStateHandle(int numNamedStates, Random random) {
	Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetaData = new HashMap<>(numNamedStates);
	long off = 0;
	for (int i = 0; i < numNamedStates; ++i) {
		long[] offsets = new long[4];
		for (int o = 0; o < offsets.length; ++o) {
			offsets[o] = off++;
		}
		OperatorStateHandle.StateMetaInfo metaInfo =
			new OperatorStateHandle.StateMetaInfo(offsets, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
		operatorStateMetaData.put(String.valueOf(UUID.randomUUID()), metaInfo);
	}
	return new OperatorStreamStateHandle(operatorStateMetaData, createStreamStateHandle(numNamedStates, random));
}

Example #25

Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0

5 votes

OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis,
		@Nullable DeserializationContext context) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}

Example #26

Source File: CoordinatorEventsExactlyOnceITCase.java From flink with Apache License 2.0

5 votes

static TaskStateSnapshot createSnapshot(StreamStateHandle handle, OperatorID operatorId) {
	final OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo(
		new long[]{0}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);

	final OperatorStateHandle state = new OperatorStreamStateHandle(
		Collections.singletonMap("état_et_moi_:_ça_fait_deux", metaInfo), handle);

	final OperatorSubtaskState oss = new OperatorSubtaskState(
		StateObjectCollection.singleton(state), StateObjectCollection.empty(),
		StateObjectCollection.empty(), StateObjectCollection.empty());
	return new TaskStateSnapshot(Collections.singletonMap(operatorId, oss));
}

Example #27

Source File: StateAssignmentOperationTest.java From flink with Apache License 2.0

5 votes

/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{52, 63}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{67, 74, 75}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{77, 88, 92}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{101, 123, 127}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
	metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{20, 27, 28}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{30, 44, 48}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{57, 79, 83}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	// rescale up case, parallelism 2 --> 3
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);

	// rescale down case, parallelism 2 --> 1
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);

	// not rescale
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}

Example #28

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}

Example #29

Source File: SavepointV2Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}

Example #30

Source File: StateAssignmentOperationTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{52, 63}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{67, 74, 75}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{77, 88, 92}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{101, 123, 127}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
	metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{20, 27, 28}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{30, 44, 48}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{57, 79, 83}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	// rescale up case, parallelism 2 --> 3
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);

	// rescale down case, parallelism 2 --> 1
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);

	// not rescale
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}