org.apache.flink.api.common.typeutils.TypeComparatorFactory Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparatorFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example #2
Source File: Utils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #3
Source File: Utils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #4
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #5
Source File: Utils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #6
Source File: IterationHeadTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 
Example #7
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private <T> TypeComparator<T> getLocalStrategyComparator(int inputNum) throws Exception {
	TypeComparatorFactory<T> compFact = this.config.getInputComparator(inputNum, getUserCodeClassLoader());
	if (compFact == null) {
		throw new Exception("Missing comparator factory for local strategy on input " + inputNum);
	}
	return compFact.createComparator();
}
 
Example #8
Source File: SingleInputPlanNode.java    From flink with Apache License 2.0 5 votes vote down vote up
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, 
		DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
{
	super(template, nodeName, driverStrategy);
	this.input = input;
	
	this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
	this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
	this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
	
	if(driverStrategy.getNumRequiredComparators() > 0) {
		this.driverKeys[0] = driverKeyFields;
		this.driverSortOrders[0] = driverSortOrders;
	}
	
	if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input.setReplicationFactor(getParallelism());
	}
	
	final PlanNode predNode = input.getSource();
	
	if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
		
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		this.branchPlan.putAll(predNode.branchPlan);
	}
}
 
Example #9
Source File: IterationHeadTask.java    From flink with Apache License 2.0 5 votes vote down vote up
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 
Example #10
Source File: TaskConfig.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 
Example #11
Source File: Utils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 
Example #12
Source File: TaskConfig.java    From flink with Apache License 2.0 5 votes vote down vote up
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 
Example #13
Source File: SynchronousChainedCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #14
Source File: SingleInputPlanNode.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, 
		DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
{
	super(template, nodeName, driverStrategy);
	this.input = input;
	
	this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
	this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
	this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
	
	if(driverStrategy.getNumRequiredComparators() > 0) {
		this.driverKeys[0] = driverKeyFields;
		this.driverSortOrders[0] = driverSortOrders;
	}
	
	if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input.setReplicationFactor(getParallelism());
	}
	
	final PlanNode predNode = input.getSource();
	
	if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
		
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		this.branchPlan.putAll(predNode.branchPlan);
	}
}
 
Example #15
Source File: GroupCombineChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #16
Source File: Utils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 
Example #17
Source File: TaskConfig.java    From flink with Apache License 2.0 5 votes vote down vote up
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 
Example #18
Source File: IterationHeadTask.java    From flink with Apache License 2.0 5 votes vote down vote up
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 
Example #19
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up
private <T> TypeComparator<T> getLocalStrategyComparator(int inputNum) throws Exception {
	TypeComparatorFactory<T> compFact = this.config.getInputComparator(inputNum, getUserCodeClassLoader());
	if (compFact == null) {
		throw new Exception("Missing comparator factory for local strategy on input " + inputNum);
	}
	return compFact.createComparator();
}
 
Example #20
Source File: JoinWithSolutionSetSecondDriver.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 
Example #21
Source File: CoGroupWithSolutionSetFirstDriver.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example #22
Source File: TaskConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public void setSolutionSetComparator(TypeComparatorFactory<?> factory) {
	setTypeComparatorFactory(factory, ITERATION_SOLUTION_SET_COMPARATOR,
		ITERATION_SOLUTION_SET_COMPARATOR_PARAMETERS);
}
 
Example #23
Source File: TaskConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public <T> TypeComparatorFactory<T> getSolutionSetComparator(ClassLoader cl) {
	return getTypeComparatorFactory(ITERATION_SOLUTION_SET_COMPARATOR,
		ITERATION_SOLUTION_SET_COMPARATOR_PARAMETERS, cl);
}
 
Example #24
Source File: CoGroupWithSolutionSetFirstDriver.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
 
Example #25
Source File: BatchTask.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder()
			.setChannelSelector(oe)
			.setTaskName(task.getEnvironment().getTaskInfo().getTaskName())
			.build(task.getEnvironment().getWriter(outputOffset + i));

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
 
Example #26
Source File: AssignRangeIndex.java    From flink with Apache License 2.0 4 votes vote down vote up
public AssignRangeIndex(TypeComparatorFactory<IN> typeComparator) {
	this.typeComparator = typeComparator;
}
 
Example #27
Source File: TaskConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public <T> TypeComparatorFactory<T> getInputComparator(int inputNum, ClassLoader cl) {
	return getTypeComparatorFactory(INPUT_STRATEGY_COMPARATOR_FACTORY_PREFIX + inputNum,
		INPUT_STRATEGY_COMPARATOR_PARAMETERS_PREFIX + inputNum + SEPARATOR, cl);
}
 
Example #28
Source File: TaskConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public void setOutputComparator(TypeComparatorFactory<?> factory, int outputNum) {
	setTypeComparatorFactory(factory, OUTPUT_TYPE_COMPARATOR_FACTORY_PREFIX + outputNum,
		OUTPUT_TYPE_COMPARATOR_PARAMETERS_PREFIX + outputNum + SEPARATOR);
}
 
Example #29
Source File: DualInputPlanNode.java    From flink with Apache License 2.0 4 votes vote down vote up
public TypeComparatorFactory<?> getComparator2() {
	return this.comparator2;
}
 
Example #30
Source File: TaskConfig.java    From flink with Apache License 2.0 4 votes vote down vote up
public void setDriverComparator(TypeComparatorFactory<?> factory, int inputNum) {
	setTypeComparatorFactory(factory, DRIVER_COMPARATOR_FACTORY_PREFIX + inputNum,
		DRIVER_COMPARATOR_PARAMETERS_PREFIX + inputNum + SEPARATOR);
}