org.apache.flink.api.common.TaskInfo Java Examples

The following examples show how to use org.apache.flink.api.common.TaskInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
		throws Exception {
	@SuppressWarnings("unchecked")
	GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
	
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}
	return typedSource.executeOnCollections(ctx, executionConfig);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: FlatMapOperatorCollectionTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: OuterJoinOperatorBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMapStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	MapStateDescriptor<String, TaskInfo> descr =
			new MapStateDescriptor<>("name", String.class, TaskInfo.class);

	context.getMapState(descr);

	MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(valueSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: CollectionExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
		throws Exception {
	@SuppressWarnings("unchecked")
	GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
	
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}
	return typedSource.executeOnCollections(ctx, executionConfig);
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: FlatMapOperatorCollectionTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: OuterJoinOperatorBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMapStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	MapStateDescriptor<String, TaskInfo> descr =
			new MapStateDescriptor<>("name", String.class, TaskInfo.class);

	context.getMapState(descr);

	MapStateDescriptor<?, ?> descrIntercepted = (MapStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> valueSerializer = descrIntercepted.getValueSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(valueSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) valueSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #14
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp = operator.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> inputData = (List<IN>) execute(inputOp, superStep);
	
	@SuppressWarnings("unchecked")
	SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractRuntimeUDFContext.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractRuntimeUDFContext(TaskInfo taskInfo,
									ClassLoader userCodeClassLoader,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?, ?>> accumulators,
									Map<String, Future<Path>> cpTasks,
									MetricGroup metrics) {
	this.taskInfo = checkNotNull(taskInfo);
	this.userCodeClassLoader = userCodeClassLoader;
	this.executionConfig = executionConfig;
	this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
	this.accumulators = checkNotNull(accumulators);
	this.metrics = metrics;
}
 
Example #17
Source Project: Flink-CEPplus   Author: ljygz   File: GenericDataSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: RichOutputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: RichInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckRuntimeContextAccess() {
	final SerializedInputFormat<Value> inputFormat = new SerializedInputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	inputFormat.setRuntimeContext(
			new RuntimeUDFContext(
					taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
					new HashMap<String, Future<Path>>(),
					new HashMap<String, Accumulator<?, ?>>(),
					new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
static Environment getMockEnvironment(File... tempDirs) {
	final String[] tempDirStrings = new String[tempDirs.length];
	for (int i = 0; i < tempDirs.length; i++) {
		tempDirStrings[i] = tempDirs[i].getAbsolutePath();
	}

	IOManager ioMan = mock(IOManager.class);
	when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);

	Environment env = mock(Environment.class);
	when(env.getJobID()).thenReturn(new JobID());
	when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
	when(env.getIOManager()).thenReturn(ioMan);
	when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

	TaskInfo taskInfo = mock(TaskInfo.class);
	when(env.getTaskInfo()).thenReturn(taskInfo);
	when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);

	TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
	when(env.getTaskManagerInfo()).thenReturn(tmInfo);

	TestTaskStateManager taskStateManager = new TestTaskStateManager();
	when(env.getTaskStateManager()).thenReturn(taskStateManager);

	return env;
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAggregatingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	AggregateFunction<String, TaskInfo, String> aggregate = (AggregateFunction<String, TaskInfo, String>) mock(AggregateFunction.class);

	AggregatingStateDescriptor<String, TaskInfo, String> descr =
			new AggregatingStateDescriptor<>("name", aggregate, TaskInfo.class);

	context.getAggregatingState(descr);

	AggregatingStateDescriptor<?, ?, ?> descrIntercepted = (AggregatingStateDescriptor<?, ?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingRuntimeContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFoldingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class);

	FoldingStateDescriptor<String, TaskInfo> descr =
			new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class);

	context.getFoldingState(descr);

	FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: StreamMockEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this.jobID = jobID;
	this.executionAttemptID = executionAttemptID;

	int subtaskIndex = 0;
	this.taskInfo = new TaskInfo(
		"", /* task name */
		1, /* num key groups / max parallelism */
		subtaskIndex, /* index of this subtask */
		1, /* num subtasks */
		0 /* attempt number */);
	this.jobConfiguration = jobConfig;
	this.taskConfiguration = taskConfig;
	this.inputs = new LinkedList<InputGate>();
	this.outputs = new LinkedList<ResultPartitionWriter>();
	this.memManager = new MemoryManager(memorySize, 1);
	this.ioManager = new IOManagerAsync();
	this.taskStateManager = Preconditions.checkNotNull(taskStateManager);
	this.aggregateManager = new TestGlobalAggregateManager();
	this.inputSplitProvider = inputSplitProvider;
	this.bufferSize = bufferSize;

	this.executionConfig = executionConfig;
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId());

	KvStateRegistry registry = new KvStateRegistry();
	this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp = operator.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> inputData = (List<IN>) execute(inputOp, superStep);
	
	@SuppressWarnings("unchecked")
	SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: AbstractRuntimeUDFContext.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractRuntimeUDFContext(TaskInfo taskInfo,
									ClassLoader userCodeClassLoader,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?, ?>> accumulators,
									Map<String, Future<Path>> cpTasks,
									MetricGroup metrics) {
	this.taskInfo = checkNotNull(taskInfo);
	this.userCodeClassLoader = userCodeClassLoader;
	this.executionConfig = executionConfig;
	this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
	this.accumulators = checkNotNull(accumulators);
	this.metrics = metrics;
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: GenericDataSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: RichOutputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}