org.apache.flink.runtime.operators.BatchTask Java Examples

The following examples show how to use org.apache.flink.runtime.operators.BatchTask. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IterationHeadTask.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriterBuilder().build(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 
Example #2
Source File: IterationHeadTask.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriter<IOReadableWritable>(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 
Example #3
Source File: IterationHeadTask.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected void initOutputs() throws Exception {
	// initialize the regular outputs first (the ones into the step function).
	super.initOutputs();

	// at this time, the outputs to the step function are created
	// add the outputs for the final solution
	List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
	final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
	final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
			userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());

	// sanity check the setup
	final int writersIntoStepFunction = this.eventualOutputs.size();
	final int writersIntoFinalResult = finalOutputWriters.size();
	final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

	if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) {
		throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
	}
	// now, we can instantiate the sync gate
	this.toSync = new RecordWriterBuilder<>().build(getEnvironment().getWriter(syncGateIndex));
	this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId();
}
 
Example #4
Source File: ChainedReduceCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;
	running = true;

	strategy = config.getDriverStrategy();

	reducer = BatchTask.instantiateUserCode(config, userCodeClassLoader, ReduceFunction.class);
	FunctionUtils.setFunctionRuntimeContext(reducer, getUdfRuntimeContext());
}
 
Example #5
Source File: SynchronousChainedCombineDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void closeTask() throws Exception {
	this.sorter.dispose();
	this.parent.getEnvironment().getMemoryManager().release(this.memory);

	if (this.running) {
		BatchTask.closeUserCode(this.combiner);
	}
}
 
Example #6
Source File: SynchronousChainedCombineDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #7
Source File: ChainedMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	final MapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 
Example #8
Source File: ChainedFlatMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final FlatMapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, FlatMapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 
Example #9
Source File: BroadcastVariableMaterialization.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private boolean decrementReferenceInternal(BatchTask<?, ?> referenceHolder, boolean errorIfNoReference) {
	synchronized (references) {
		if (disposed || references.isEmpty()) {
			if (errorIfNoReference) {
				throw new IllegalStateException("Decrementing reference to broadcast variable that is no longer alive.");
			} else {
				return false;
			}
		}

		if (!references.remove(referenceHolder)) {
			if (errorIfNoReference) {
				throw new IllegalStateException(
						String.format("The task %s did not hold a reference to the broadcast variable %s.",
								referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(),
								key.toString()));
			} else {
				return false;
			}
		}

		if (references.isEmpty()) {
			disposed = true;
			data = null;
			transformed = null;
			return true;
		} else {
			return false;
		}
	}
}
 
Example #10
Source File: SynchronousChainedCombineDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupCombineFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupCombineFunction.class);
	this.combiner = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 
Example #11
Source File: ChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics, env.getExternalResourceInfoProvider()
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
 
Example #12
Source File: GroupCombineChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #13
Source File: ChainTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchTaskOutputInCloseMethod() {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	try {
		initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
		addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
		addOutput(outList);
		registerTask(FlatMapDriver.class, MockMapStub.class);
		for (int i = 0; i < numChainedTasks; i++) {
			final TaskConfig taskConfig = new TaskConfig(new Configuration());
			taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
			taskConfig.setOutputSerializer(serFact);
			taskConfig.setStubWrapper(
				new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
			getTaskConfig().addChainedTask(
				ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
		}
		final BatchTask<FlatMapFunction<Record, Record>, Record> testTask =
			new BatchTask<>(mockEnv);
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #14
Source File: ChainedReduceCombineDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void closeTask() throws Exception {
	if (sorter != null) {
		sorter.dispose();
	}
	if (table != null) {
		table.close();
	}
	parent.getEnvironment().getMemoryManager().release(memory);
	BatchTask.closeUserCode(reducer);
}
 
Example #15
Source File: GroupCombineChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupReduceFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
	this.reducer = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 
Example #16
Source File: BroadcastVariableManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void releaseAllReferencesFromTask(BatchTask<?, ?> referenceHolder) {
	// go through all registered variables
	for (Map.Entry<BroadcastVariableKey, BroadcastVariableMaterialization<?, ?>> entry : variables.entrySet()) {
		BroadcastVariableMaterialization<?, ?> mat = entry.getValue();

		// release the reference
		if (mat.decrementReferenceIfHeld(referenceHolder)) {
			// remove if no one holds a reference and no one concurrently replaced the entry
			variables.remove(entry.getKey(), mat);
		}
	}
}
 
Example #17
Source File: ExecutionGraphDeploymentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecution(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
	v1.setParallelism(dop1);
	v2.setParallelism(dop2);

	v1.setInvokableClass(BatchTask.class);
	v2.setInvokableClass(BatchTask.class);

	final ArrayDeque<CompletableFuture<LogicalSlot>> slotFutures = new ArrayDeque<>();
	for (int i = 0; i < dop1 + dop2; i++) {
		slotFutures.addLast(CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot()));
	}

	final SlotProvider slotProvider = new TestingSlotProvider(ignore -> slotFutures.removeFirst());

	DirectScheduledExecutorService executorService = new DirectScheduledExecutorService();

	// execution graph that executes actions synchronously
	ExecutionGraph eg = TestingExecutionGraphBuilder
		.newBuilder()
		.setFutureExecutor(executorService)
		.setSlotProvider(slotProvider)
		.setBlobWriter(blobWriter)
		.build();

	checkJobOffloaded(eg);

	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<JobVertex> ordered = Arrays.asList(v1, v2);
	eg.attachJobGraph(ordered);

	// schedule, this triggers mock deployment
	eg.scheduleForExecution();

	Map<ExecutionAttemptID, Execution> executions = eg.getRegisteredExecutions();
	assertEquals(dop1 + dop2, executions.size());

	return new Tuple2<>(eg, executions);
}
 
Example #18
Source File: BroadcastVariableMaterialization.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean decrementReferenceInternal(BatchTask<?, ?> referenceHolder, boolean errorIfNoReference) {
	synchronized (references) {
		if (disposed || references.isEmpty()) {
			if (errorIfNoReference) {
				throw new IllegalStateException("Decrementing reference to broadcast variable that is no longer alive.");
			} else {
				return false;
			}
		}

		if (!references.remove(referenceHolder)) {
			if (errorIfNoReference) {
				throw new IllegalStateException(
						String.format("The task %s did not hold a reference to the broadcast variable %s.",
								referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(),
								key.toString()));
			} else {
				return false;
			}
		}

		if (references.isEmpty()) {
			disposed = true;
			data = null;
			transformed = null;
			return true;
		} else {
			return false;
		}
	}
}
 
Example #19
Source File: ChainedReduceCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void closeTask() throws Exception {
	if (sorter != null) {
		sorter.dispose();
	}
	if (table != null) {
		table.close();
	}
	parent.getEnvironment().getMemoryManager().release(memory);
	BatchTask.closeUserCode(reducer);
}
 
Example #20
Source File: BroadcastVariableManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void releaseReference(BroadcastVariableKey key, BatchTask<?, ?> referenceHolder) {
	BroadcastVariableMaterialization<?, ?> mat = variables.get(key);

	// release this reference
	if (mat.decrementReference(referenceHolder)) {
		// remove if no one holds a reference and no one concurrently replaced the entry
		variables.remove(key, mat);
	}
}
 
Example #21
Source File: BroadcastVariableManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void releaseAllReferencesFromTask(BatchTask<?, ?> referenceHolder) {
	// go through all registered variables
	for (Map.Entry<BroadcastVariableKey, BroadcastVariableMaterialization<?, ?>> entry : variables.entrySet()) {
		BroadcastVariableMaterialization<?, ?> mat = entry.getValue();

		// release the reference
		if (mat.decrementReferenceIfHeld(referenceHolder)) {
			// remove if no one holds a reference and no one concurrently replaced the entry
			variables.remove(entry.getKey(), mat);
		}
	}
}
 
Example #22
Source File: ChainedAllReduceDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final ReduceFunction<IT> red = BatchTask.instantiateUserCode(this.config, userCodeClassLoader, ReduceFunction.class);
	this.reducer = red;
	FunctionUtils.setFunctionRuntimeContext(red, getUdfRuntimeContext());

	TypeSerializerFactory<IT> serializerFactory = this.config.getInputSerializer(0, userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ChainedAllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #23
Source File: JobGraphGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
	final String taskName = node.getNodeName();
	final DriverStrategy ds = node.getDriverStrategy();
	final JobVertex vertex = new JobVertex(taskName);
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());
	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class);
	
	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());
	
	// set the driver strategy
	config.setDriver(ds.getDriverClass());
	config.setDriverStrategy(ds);
	if (node.getComparator1() != null) {
		config.setDriverComparator(node.getComparator1(), 0);
	}
	if (node.getComparator2() != null) {
		config.setDriverComparator(node.getComparator2(), 1);
	}
	if (node.getPairComparator() != null) {
		config.setDriverPairComparator(node.getPairComparator());
	}
	
	// assign memory, file-handles, etc.
	assignDriverResources(node, config);
	return vertex;
}
 
Example #24
Source File: BroadcastVariableMaterialization.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean decrementReferenceInternal(BatchTask<?, ?> referenceHolder, boolean errorIfNoReference) {
	synchronized (references) {
		if (disposed || references.isEmpty()) {
			if (errorIfNoReference) {
				throw new IllegalStateException("Decrementing reference to broadcast variable that is no longer alive.");
			} else {
				return false;
			}
		}

		if (!references.remove(referenceHolder)) {
			if (errorIfNoReference) {
				throw new IllegalStateException(
						String.format("The task %s did not hold a reference to the broadcast variable %s.",
								referenceHolder.getEnvironment().getTaskInfo().getTaskNameWithSubtasks(),
								key.toString()));
			} else {
				return false;
			}
		}

		if (references.isEmpty()) {
			disposed = true;
			data = null;
			transformed = null;
			return true;
		} else {
			return false;
		}
	}
}
 
Example #25
Source File: ExecutionGraphDeploymentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecution(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
	v1.setParallelism(dop1);
	v2.setParallelism(dop2);

	v1.setInvokableClass(BatchTask.class);
	v2.setInvokableClass(BatchTask.class);

	final ArrayDeque<CompletableFuture<LogicalSlot>> slotFutures = new ArrayDeque<>();
	for (int i = 0; i < dop1 + dop2; i++) {
		slotFutures.addLast(CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot()));
	}

	final SlotProvider slotProvider = new TestingSlotProvider(ignore -> slotFutures.removeFirst());

	DirectScheduledExecutorService executorService = new DirectScheduledExecutorService();

	// execution graph that executes actions synchronously
	ExecutionGraph eg = createExecutionGraphWithoutQueuedScheduling(new JobID(), slotProvider, executorService, TestingUtils.defaultExecutor());
	checkJobOffloaded(eg);

	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<JobVertex> ordered = Arrays.asList(v1, v2);
	eg.attachJobGraph(ordered);

	// schedule, this triggers mock deployment
	eg.scheduleForExecution();

	Map<ExecutionAttemptID, Execution> executions = eg.getRegisteredExecutions();
	assertEquals(dop1 + dop2, executions.size());

	return new Tuple2<>(eg, executions);
}
 
Example #26
Source File: SynchronousChainedCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupCombineFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupCombineFunction.class);
	this.combiner = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 
Example #27
Source File: SynchronousChainedCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #28
Source File: SynchronousChainedCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void closeTask() throws Exception {
	this.sorter.dispose();
	this.parent.getEnvironment().getMemoryManager().release(this.memory);

	if (this.running) {
		BatchTask.closeUserCode(this.combiner);
	}
}
 
Example #29
Source File: ChainedFlatMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final FlatMapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, FlatMapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 
Example #30
Source File: ChainedReduceCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = config.getStubParameters();
	BatchTask.openUserCode(reducer, stubConfig);

	// instantiate the serializer / comparator
	serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer();
	comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator();

	MemoryManager memManager = parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
	memory = memManager.allocatePages(parent, numMemoryPages);

	LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");

	switch (strategy) {
		case SORTED_PARTIAL_REDUCE:
			// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
			if (comparator.supportsSerializationWithKeyNormalization() &&
				serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
				sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
			} else {
				sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
			}
			break;
		case HASHED_PARTIAL_REDUCE:
			table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
			table.open();
			reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled);
			break;
	}
}