Java Code Examples for org.apache.flink.api.common.ExecutionConfig#isObjectReuseEnabled()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#isObjectReuseEnabled() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: GroupReduceDriver.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
		throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
	}
	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 2
Source Project: flink   File: AllReduceDriver.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 3
Source Project: Flink-CEPplus   File: ReduceDriver.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
		throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
	}
	this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 4
Source Project: flink   File: FlatMapDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("FlatMapDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 5
Source Project: flink   File: ChainedDriver.java    License: Apache License 2.0 5 votes vote down vote up
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics, env.getExternalResourceInfoProvider()
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
 
Example 6
Source Project: Flink-CEPplus   File: FlatMapDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("FlatMapDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 7
Source Project: Flink-CEPplus   File: MapDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup(TaskContext<MapFunction<IT, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
}
 
Example 8
Source Project: flink   File: NoOpDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("NoOpDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 9
Source Project: flink   File: AllGroupReduceDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	this.strategy = config.getDriverStrategy();

	switch (this.strategy) {
		case ALL_GROUP_REDUCE_COMBINE:
			if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
			}
		case ALL_GROUP_REDUCE:
		case ALL_GROUP_COMBINE:
			break;
		default:
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
	}

	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 10
Source Project: Flink-CEPplus   File: ChainedDriver.java    License: Apache License 2.0 5 votes vote down vote up
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().getOrAddOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
 
Example 11
Source Project: flink   File: MapDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup(TaskContext<MapFunction<IT, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
}
 
Example 12
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 13
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 14
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example 15
@Override
public void initialize() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();

	final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	MutableObjectIterator<IT1> input1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	MutableObjectIterator<IT2> input2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);

	TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = 
			this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());

	double availableMemory = config.getRelativeMemoryDriver();
	boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration()
		.getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		if (buildSideIndex == 0 && probeSideIndex == 1) {

			matchIterator = new ReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);


		} else if (buildSideIndex == 1 && probeSideIndex == 0) {

			matchIterator = new ReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);

		} else {
			throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
		}
	} else {
		if (buildSideIndex == 0 && probeSideIndex == 1) {

			matchIterator = new NonReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);


		} else if (buildSideIndex == 1 && probeSideIndex == 0) {

			matchIterator = new NonReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);

		} else {
			throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
		}
	}
	
	this.matchIterator.open();
}
 
Example 16
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 17
Source Project: flink   File: AbstractOuterJoinDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}
 
Example 18
@Override
public void initialize() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();

	final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	MutableObjectIterator<IT1> input1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	MutableObjectIterator<IT2> input2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);

	TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = 
			this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());

	double availableMemory = config.getRelativeMemoryDriver();
	boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration()
		.getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		if (buildSideIndex == 0 && probeSideIndex == 1) {

			matchIterator = new ReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);


		} else if (buildSideIndex == 1 && probeSideIndex == 0) {

			matchIterator = new ReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);

		} else {
			throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
		}
	} else {
		if (buildSideIndex == 0 && probeSideIndex == 1) {

			matchIterator = new NonReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);


		} else if (buildSideIndex == 1 && probeSideIndex == 0) {

			matchIterator = new NonReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
					input1, input2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					this.taskContext.getMemoryManager(),
					this.taskContext.getIOManager(),
					this.taskContext.getContainingTask(),
					availableMemory,
					false,
					false,
					hashJoinUseBitMaps);

		} else {
			throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
		}
	}
	
	this.matchIterator.open();
}
 
Example 19
Source Project: flink   File: AbstractOuterJoinDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}
 
Example 20
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}