Java Code Examples for org.apache.flink.api.common.typeutils.TypeComparator

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: BatchTask.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example 2
Source Project: flink   Source File: KeySelectorUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static <X> KeySelector<X, Tuple> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo, ExecutionConfig executionConfig) {
	if (!(typeInfo instanceof CompositeType)) {
		throw new InvalidTypesException(
				"This key operation requires a composite type such as Tuples, POJOs, or Case Classes.");
	}

	CompositeType<X> compositeType = (CompositeType<X>) typeInfo;

	int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
	int numKeyFields = logicalKeyPositions.length;

	TypeInformation<?>[] typeInfos = keys.getKeyFieldTypes();
	// use ascending order here, the code paths for that are usually a slight bit faster
	boolean[] orders = new boolean[numKeyFields];
	for (int i = 0; i < numKeyFields; i++) {
		orders[i] = true;
	}

	TypeComparator<X> comparator = compositeType.createComparator(logicalKeyPositions, orders, 0, executionConfig);
	return new ComparableKeySelector<>(comparator, numKeyFields, new TupleTypeInfo<>(typeInfos));
}
 
Example 3
Source Project: Flink-CEPplus   Source File: RowTypeInfo.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeComparator<Row> createComparator(
	int[] logicalKeyFields,
	boolean[] orders,
	int logicalFieldOffset,
	ExecutionConfig config) {

	comparatorOrders = orders;
	TypeComparator<Row> comparator = super.createComparator(
		logicalKeyFields,
		orders,
		logicalFieldOffset,
		config);
	comparatorOrders = null;
	return comparator;
}
 
Example 4
Source Project: flink   Source File: RowComparator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int compareToReference(TypeComparator<Row> referencedComparator) {
	RowComparator other = (RowComparator) referencedComparator;
	int i = 0;
	try {
		for (; i < keyPositions.length; i++) {
			int cmp = comparators[i].compareToReference(other.comparators[i]);
			if (cmp != 0) {
				return cmp;
			}
		}
	} catch (IndexOutOfBoundsException e) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i]);
	}
	return 0;
}
 
Example 5
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example 6
Source Project: flink   Source File: PojoComparator.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private PojoComparator(PojoComparator<T> toClone) {
	this.keyFields = toClone.keyFields;
	this.comparators = new TypeComparator[toClone.comparators.length];

	for (int i = 0; i < toClone.comparators.length; i++) {
		this.comparators[i] = toClone.comparators[i].duplicate();
	}

	this.normalizedKeyLengths = toClone.normalizedKeyLengths;
	this.numLeadingNormalizableKeys = toClone.numLeadingNormalizableKeys;
	this.normalizableKeyPrefixLen = toClone.normalizableKeyPrefixLen;
	this.invertNormKey = toClone.invertNormKey;

	this.type = toClone.type;

	try {
		this.serializer = (TypeSerializer<T>) InstantiationUtil.deserializeObject(
				InstantiationUtil.serializeObject(toClone.serializer), Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException("Cannot copy serializer", e);
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example 8
Source Project: flink   Source File: AbstractMergeOuterJoinIterator.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.outerJoinType = outerJoinType;
}
 
Example 9
Source Project: flink   Source File: BatchTask.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example 10
Source Project: flink   Source File: JavaApiPostPass.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example 11
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example 12
Source Project: flink   Source File: ReusingSortMergeCoGroupIterator.java    License: Apache License 2.0 5 votes vote down vote up
public ReusingSortMergeCoGroupIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2, TypeSerializer<T1>
		serializer1, TypeComparator<T1> groupingComparator1,
		TypeSerializer<T2> serializer2,
		TypeComparator<T2> groupingComparator2,
		TypePairComparator<T1, T2> pairComparator)
{

	this.comp = pairComparator;

	this.iterator1 = new ReusingKeyGroupedIterator<T1>(input1, serializer1, groupingComparator1);
	this.iterator2 = new ReusingKeyGroupedIterator<T2>(input2, serializer2, groupingComparator2);
}
 
Example 13
Source Project: Flink-CEPplus   Source File: GroupCombineChainedDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 14
Source Project: flink   Source File: TupleComparatorILDC3Test.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 0, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 
Example 15
Source Project: Flink-CEPplus   Source File: ValueArrayTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public TypeComparator<ValueArray<T>> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) {
	Preconditions.checkNotNull(type, "TypeInformation type class is required");

	if (ByteValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new ByteValueArrayComparator(sortOrderAscending);
	} else if (CharValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new CharValueArrayComparator(sortOrderAscending);
	} else if (DoubleValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new DoubleValueArrayComparator(sortOrderAscending);
	} else if (FloatValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new FloatValueArrayComparator(sortOrderAscending);
	} else if (IntValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new IntValueArrayComparator(sortOrderAscending);
	} else if (LongValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new LongValueArrayComparator(sortOrderAscending);
	} else if (NullValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new NullValueArrayComparator(sortOrderAscending);
	} else if (ShortValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new ShortValueArrayComparator(sortOrderAscending);
	} else if (StringValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new StringValueArrayComparator(sortOrderAscending);
	} else {
		throw new InvalidTypesException("No ValueArray class exists for " + type);
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: TupleComparatorILD3Test.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1, 2},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending),
				new DoubleComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 
Example 17
Source Project: Flink-CEPplus   Source File: CommonRangeBoundaries.java    License: Apache License 2.0 5 votes vote down vote up
private int compareKeys(TypeComparator[] flatComparators, Object[] keys, Object[] boundary) {
	if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
		throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
	}

	for (int i=0; i<flatComparators.length; i++) {
		int result = flatComparators[i].compare(keys[i], boundary[i]);
		if (result != 0) {
			return result;
		}
	}

	return 0;
}
 
Example 18
Source Project: flink   Source File: UnilateralSortMerger.java    License: Apache License 2.0 5 votes vote down vote up
public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 
Example 19
Source Project: flink   Source File: OutputEmitterTest.java    License: Apache License 2.0 5 votes vote down vote up
private int[] getSelectedChannelsHitCount(
		ShipStrategyType shipStrategyType,
		int numRecords,
		int numberOfChannels,
		Enum recordType) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {0}, new Class[] {recordType == RecordType.INTEGER ? IntValue.class : StringValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}
 
Example 20
Source Project: flink   Source File: FixedLengthRecordSorter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 
Example 21
Source Project: flink   Source File: GroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllReduceDriverAccumulatingImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableAccumulatingReducer());
		context.setMutableObjectMode(false);
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 22
public NonReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1,
			comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example 23
Source Project: flink   Source File: RowDataTypeInfo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeComparator<RowData> createComparator(
	int[] logicalKeyFields,
	boolean[] orders,
	int logicalFieldOffset,
	ExecutionConfig config) {
	// TODO support it.
	throw new UnsupportedOperationException("Not support yet!");
}
 
Example 24
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType,
MutableObjectIterator<T1> input1,
MutableObjectIterator<T2> input2,
TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
TypePairComparator<T1, T2> pairComparator,
MemoryManager memoryManager,
IOManager ioManager,
int numMemoryPages,
AbstractInvokable parentTask) throws Exception;
 
Example 25
Source Project: flink   Source File: CommonRangeBoundaries.java    License: Apache License 2.0 5 votes vote down vote up
private int compareKeys(TypeComparator[] flatComparators, Object[] keys, Object[] boundary) {
	if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
		throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
	}

	for (int i=0; i<flatComparators.length; i++) {
		int result = flatComparators[i].compare(keys[i], boundary[i]);
		if (result != 0) {
			return result;
		}
	}

	return 0;
}
 
Example 26
public NonReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1,
			comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example 27
Source Project: Flink-CEPplus   Source File: TupleComparatorILDXC2Test.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, DoubleSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 
Example 28
Source Project: flink   Source File: PojoComparator.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void getFlatComparator(List<TypeComparator> flatComparators) {
	for(int i = 0; i < comparators.length; i++) {
		if(comparators[i] instanceof CompositeTypeComparator) {
			((CompositeTypeComparator)comparators[i]).getFlatComparator(flatComparators);
		} else {
			flatComparators.add(comparators[i]);
		}
	}
}
 
Example 29
Source Project: flink   Source File: LeftOuterJoinDriver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
		DriverStrategy driverStrategy,
		MutableObjectIterator<IT1> in1,
		MutableObjectIterator<IT2> in2,
		TypeSerializer<IT1> serializer1,
		TypeComparator<IT1> comparator1,
		TypeSerializer<IT2> serializer2,
		TypeComparator<IT2> comparator2,
		TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
		MemoryManager memoryManager,
		IOManager ioManager,
		double driverMemFraction
) throws Exception {
	switch (driverStrategy) {
		case LEFT_OUTER_MERGE:
			int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
			return new NonReusingMergeOuterJoinIterator<>(
					OuterJoinType.LEFT,
					in1,
					in2,
					serializer1,
					comparator1,
					serializer2,
					comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					memoryManager,
					ioManager,
					numPages,
					super.taskContext.getContainingTask()
			);
		case LEFT_HYBRIDHASH_BUILD_FIRST:
			return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					memoryManager, ioManager,
					this.taskContext.getContainingTask(),
					driverMemFraction,
					false,
					true,
					false);
		case LEFT_HYBRIDHASH_BUILD_SECOND:
			return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					memoryManager, ioManager,
					this.taskContext.getContainingTask(),
					driverMemFraction,
					true,
					false,
					false);
		default:
			throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: TestData.java    License: Apache License 2.0 4 votes vote down vote up
public static TypeComparator<Tuple2<Integer, String>> getIntStringTupleComparator() {
	return getIntStringTupleTypeInfo().createComparator(new int[]{0}, new boolean[]{true}, 0, null);
}