Java Code Examples for org.apache.flink.runtime.operators.hash.CompactingHashTable

The following examples show how to use org.apache.flink.runtime.operators.hash.CompactingHashTable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: AbstractIterativeTask.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new solution set update output collector.
 *
 * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the
 * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing)
 * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing)
 * {@link SolutionSetUpdateOutputCollector} is created.
 *
 * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call
 * collect(T) of the delegate.
 *
 * @param delegate null -OR- a delegate collector to be called by the newly created collector
 * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or
 * {@link SolutionSetUpdateOutputCollector}
 */
protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) {
	Broker<Object> solutionSetBroker = SolutionSetBroker.instance();

	Object ss = solutionSetBroker.get(brokerKey());
	if (ss instanceof CompactingHashTable) {
		@SuppressWarnings("unchecked")
		CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss;
		return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate);
	}
	else if (ss instanceof JoinHashMap) {
		@SuppressWarnings("unchecked")
		JoinHashMap<OT> map = (JoinHashMap<OT>) ss;
		return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate);
	} else {
		throw new RuntimeException("Unrecognized solution set handle: " + ss);
	}
}
 
Example 2
Source Project: flink   Source File: AbstractIterativeTask.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new solution set update output collector.
 *
 * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the
 * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing)
 * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing)
 * {@link SolutionSetUpdateOutputCollector} is created.
 *
 * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call
 * collect(T) of the delegate.
 *
 * @param delegate null -OR- a delegate collector to be called by the newly created collector
 * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or
 * {@link SolutionSetUpdateOutputCollector}
 */
protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) {
	Broker<Object> solutionSetBroker = SolutionSetBroker.instance();

	Object ss = solutionSetBroker.get(brokerKey());
	if (ss instanceof CompactingHashTable) {
		@SuppressWarnings("unchecked")
		CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss;
		return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate);
	}
	else if (ss instanceof JoinHashMap) {
		@SuppressWarnings("unchecked")
		JoinHashMap<OT> map = (JoinHashMap<OT>) ss;
		return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate);
	} else {
		throw new RuntimeException("Unrecognized solution set handle: " + ss);
	}
}
 
Example 3
Source Project: flink   Source File: AbstractIterativeTask.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new solution set update output collector.
 *
 * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the
 * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing)
 * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing)
 * {@link SolutionSetUpdateOutputCollector} is created.
 *
 * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call
 * collect(T) of the delegate.
 *
 * @param delegate null -OR- a delegate collector to be called by the newly created collector
 * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or
 * {@link SolutionSetUpdateOutputCollector}
 */
protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) {
	Broker<Object> solutionSetBroker = SolutionSetBroker.instance();

	Object ss = solutionSetBroker.get(brokerKey());
	if (ss instanceof CompactingHashTable) {
		@SuppressWarnings("unchecked")
		CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss;
		return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate);
	}
	else if (ss instanceof JoinHashMap) {
		@SuppressWarnings("unchecked")
		JoinHashMap<OT> map = (JoinHashMap<OT>) ss;
		return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate);
	} else {
		throw new RuntimeException("Unrecognized solution set handle: " + ss);
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: IterationHeadTask.java    License: Apache License 2.0 5 votes vote down vote up
private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException {
	final MutableObjectIterator<X> results = hashTable.getEntryIterator();
	final Collector<X> output = this.finalOutputCollector;
	X record = solutionTypeSerializer.getSerializer().createInstance();

	while ((record = results.next(record)) != null) {
		output.collect(record);
	}
}
 
Example 5
Source Project: flink   Source File: IterationHeadTask.java    License: Apache License 2.0 5 votes vote down vote up
private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException {
	final MutableObjectIterator<X> results = hashTable.getEntryIterator();
	final Collector<X> output = this.finalOutputCollector;
	X record = solutionTypeSerializer.getSerializer().createInstance();

	while ((record = results.next(record)) != null) {
		output.collect(record);
	}
}
 
Example 6
Source Project: flink   Source File: IterationHeadTask.java    License: Apache License 2.0 5 votes vote down vote up
private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException {
	final MutableObjectIterator<X> results = hashTable.getEntryIterator();
	final Collector<X> output = this.finalOutputCollector;
	X record = solutionTypeSerializer.getSerializer().createInstance();

	while ((record = results.next(record)) != null) {
		output.collect(record);
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: IterationHeadTask.java    License: Apache License 2.0 4 votes vote down vote up
private void readInitialSolutionSet(CompactingHashTable<X> solutionSet, MutableObjectIterator<X> solutionSetInput) throws IOException {
	solutionSet.open();
	solutionSet.buildTableWithUniqueKey(solutionSetInput);
}
 
Example 8
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 9
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 10
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 11
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 12
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	};
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 13
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 14
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example 15
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeSerializer<IT1> solutionSetSerializer;
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}

	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example 16
Source Project: flink   Source File: IterationHeadTask.java    License: Apache License 2.0 4 votes vote down vote up
private void readInitialSolutionSet(CompactingHashTable<X> solutionSet, MutableObjectIterator<X> solutionSetInput) throws IOException {
	solutionSet.open();
	solutionSet.buildTableWithUniqueKey(solutionSetInput);
}
 
Example 17
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 18
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 19
Source Project: flink   Source File: SolutionSetUpdateOutputCollector.java    License: Apache License 2.0 4 votes vote down vote up
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 20
Source Project: flink   Source File: SolutionSetUpdateOutputCollector.java    License: Apache License 2.0 4 votes vote down vote up
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 21
Source Project: flink   Source File: CoGroupWithSolutionSetSecondDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	};
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 22
Source Project: flink   Source File: JoinWithSolutionSetSecondDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example 23
Source Project: flink   Source File: CoGroupWithSolutionSetFirstDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example 24
Source Project: flink   Source File: JoinWithSolutionSetFirstDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeSerializer<IT1> solutionSetSerializer;
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}

	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example 25
Source Project: flink   Source File: IterationHeadTask.java    License: Apache License 2.0 4 votes vote down vote up
private void readInitialSolutionSet(CompactingHashTable<X> solutionSet, MutableObjectIterator<X> solutionSetInput) throws IOException {
	solutionSet.open();
	solutionSet.buildTableWithUniqueKey(solutionSetInput);
}
 
Example 26
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 27
public SolutionSetFastUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 28
Source Project: flink   Source File: SolutionSetUpdateOutputCollector.java    License: Apache License 2.0 4 votes vote down vote up
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet) {
	this(solutionSet, null);
}
 
Example 29
Source Project: flink   Source File: SolutionSetUpdateOutputCollector.java    License: Apache License 2.0 4 votes vote down vote up
public SolutionSetUpdateOutputCollector(CompactingHashTable<T> solutionSet, Collector<T> delegate) {
	this.solutionSet = solutionSet;
	this.delegate = delegate;
}
 
Example 30
Source Project: flink   Source File: CoGroupWithSolutionSetSecondDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	};
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}