org.apache.flink.api.common.typeutils.TypeComparator Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example #2
Source File: KeySelectorUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
public static <X> KeySelector<X, Tuple> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo, ExecutionConfig executionConfig) {
	if (!(typeInfo instanceof CompositeType)) {
		throw new InvalidTypesException(
				"This key operation requires a composite type such as Tuples, POJOs, or Case Classes.");
	}

	CompositeType<X> compositeType = (CompositeType<X>) typeInfo;

	int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
	int numKeyFields = logicalKeyPositions.length;

	TypeInformation<?>[] typeInfos = keys.getKeyFieldTypes();
	// use ascending order here, the code paths for that are usually a slight bit faster
	boolean[] orders = new boolean[numKeyFields];
	for (int i = 0; i < numKeyFields; i++) {
		orders[i] = true;
	}

	TypeComparator<X> comparator = compositeType.createComparator(logicalKeyPositions, orders, 0, executionConfig);
	return new ComparableKeySelector<>(comparator, numKeyFields, new TupleTypeInfo<>(typeInfos));
}
 
Example #3
Source File: RowTypeInfo.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public TypeComparator<Row> createComparator(
	int[] logicalKeyFields,
	boolean[] orders,
	int logicalFieldOffset,
	ExecutionConfig config) {

	comparatorOrders = orders;
	TypeComparator<Row> comparator = super.createComparator(
		logicalKeyFields,
		orders,
		logicalFieldOffset,
		config);
	comparatorOrders = null;
	return comparator;
}
 
Example #4
Source File: RowComparator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public int compareToReference(TypeComparator<Row> referencedComparator) {
	RowComparator other = (RowComparator) referencedComparator;
	int i = 0;
	try {
		for (; i < keyPositions.length; i++) {
			int cmp = comparators[i].compareToReference(other.comparators[i]);
			if (cmp != 0) {
				return cmp;
			}
		}
	} catch (IndexOutOfBoundsException e) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i]);
	}
	return 0;
}
 
Example #5
Source File: ReusingBuildSecondReOpenableHashJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example #6
Source File: PojoComparator.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private PojoComparator(PojoComparator<T> toClone) {
	this.keyFields = toClone.keyFields;
	this.comparators = new TypeComparator[toClone.comparators.length];

	for (int i = 0; i < toClone.comparators.length; i++) {
		this.comparators[i] = toClone.comparators[i].duplicate();
	}

	this.normalizedKeyLengths = toClone.normalizedKeyLengths;
	this.numLeadingNormalizableKeys = toClone.numLeadingNormalizableKeys;
	this.normalizableKeyPrefixLen = toClone.normalizableKeyPrefixLen;
	this.invertNormKey = toClone.invertNormKey;

	this.type = toClone.type;

	try {
		this.serializer = (TypeSerializer<T>) InstantiationUtil.deserializeObject(
				InstantiationUtil.serializeObject(toClone.serializer), Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException("Cannot copy serializer", e);
	}
}
 
Example #7
Source File: UnilateralSortMerger.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example #8
Source File: AbstractMergeOuterJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
public AbstractMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.outerJoinType = outerJoinType;
}
 
Example #9
Source File: BatchTask.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example #10
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #11
Source File: NonReusingBuildSecondReOpenableHashJoinIterator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example #12
Source File: ReusingSortMergeCoGroupIterator.java    From flink with Apache License 2.0 5 votes vote down vote up
public ReusingSortMergeCoGroupIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2, TypeSerializer<T1>
		serializer1, TypeComparator<T1> groupingComparator1,
		TypeSerializer<T2> serializer2,
		TypeComparator<T2> groupingComparator2,
		TypePairComparator<T1, T2> pairComparator)
{

	this.comp = pairComparator;

	this.iterator1 = new ReusingKeyGroupedIterator<T1>(input1, serializer1, groupingComparator1);
	this.iterator2 = new ReusingKeyGroupedIterator<T2>(input2, serializer2, groupingComparator2);
}
 
Example #13
Source File: GroupCombineChainedDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #14
Source File: TupleComparatorILDC3Test.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 0, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new IntComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 
Example #15
Source File: ValueArrayTypeInfo.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public TypeComparator<ValueArray<T>> createComparator(boolean sortOrderAscending, ExecutionConfig executionConfig) {
	Preconditions.checkNotNull(type, "TypeInformation type class is required");

	if (ByteValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new ByteValueArrayComparator(sortOrderAscending);
	} else if (CharValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new CharValueArrayComparator(sortOrderAscending);
	} else if (DoubleValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new DoubleValueArrayComparator(sortOrderAscending);
	} else if (FloatValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new FloatValueArrayComparator(sortOrderAscending);
	} else if (IntValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new IntValueArrayComparator(sortOrderAscending);
	} else if (LongValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new LongValueArrayComparator(sortOrderAscending);
	} else if (NullValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new NullValueArrayComparator(sortOrderAscending);
	} else if (ShortValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new ShortValueArrayComparator(sortOrderAscending);
	} else if (StringValue.class.isAssignableFrom(type)) {
		return (TypeComparator<ValueArray<T>>) (TypeComparator<?>) new StringValueArrayComparator(sortOrderAscending);
	} else {
		throw new InvalidTypesException("No ValueArray class exists for " + type);
	}
}
 
Example #16
Source File: TupleComparatorILD3Test.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{0, 1, 2},
			new TypeComparator[]{
				new IntComparator(ascending),
				new LongComparator(ascending),
				new DoubleComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, LongSerializer.INSTANCE, DoubleSerializer.INSTANCE });
}
 
Example #17
Source File: CommonRangeBoundaries.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private int compareKeys(TypeComparator[] flatComparators, Object[] keys, Object[] boundary) {
	if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
		throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
	}

	for (int i=0; i<flatComparators.length; i++) {
		int result = flatComparators[i].compare(keys[i], boundary[i]);
		if (result != 0) {
			return result;
		}
	}

	return 0;
}
 
Example #18
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 
Example #19
Source File: OutputEmitterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private int[] getSelectedChannelsHitCount(
		ShipStrategyType shipStrategyType,
		int numRecords,
		int numberOfChannels,
		Enum recordType) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {0}, new Class[] {recordType == RecordType.INTEGER ? IntValue.class : StringValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}
 
Example #20
Source File: FixedLengthRecordSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 
Example #21
Source File: GroupReduceDriverTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testAllReduceDriverAccumulatingImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableAccumulatingReducer());
		context.setMutableObjectMode(false);
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #22
Source File: NonReusingBuildSecondHashJoinIterator.java    From flink with Apache License 2.0 5 votes vote down vote up
public NonReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1,
			comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example #23
Source File: RowDataTypeInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeComparator<RowData> createComparator(
	int[] logicalKeyFields,
	boolean[] orders,
	int logicalFieldOffset,
	ExecutionConfig config) {
	// TODO support it.
	throw new UnsupportedOperationException("Not support yet!");
}
 
Example #24
Source File: AbstractSortMergeOuterJoinIteratorITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
protected abstract <T1, T2, T3> AbstractMergeOuterJoinIterator<T1, T2, T3> createOuterJoinIterator(OuterJoinType outerJoinType,
MutableObjectIterator<T1> input1,
MutableObjectIterator<T2> input2,
TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
TypePairComparator<T1, T2> pairComparator,
MemoryManager memoryManager,
IOManager ioManager,
int numMemoryPages,
AbstractInvokable parentTask) throws Exception;
 
Example #25
Source File: CommonRangeBoundaries.java    From flink with Apache License 2.0 5 votes vote down vote up
private int compareKeys(TypeComparator[] flatComparators, Object[] keys, Object[] boundary) {
	if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
		throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
	}

	for (int i=0; i<flatComparators.length; i++) {
		int result = flatComparators[i].compare(keys[i], boundary[i]);
		if (result != 0) {
			return result;
		}
	}

	return 0;
}
 
Example #26
Source File: NonReusingBuildSecondHashJoinIterator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public NonReusingBuildSecondHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	this.memManager = memManager;
	this.firstInput = firstInput;
	this.secondInput = secondInput;
	this.probeSideSerializer = serializer1;

	if(useBitmapFilters && probeSideOuterJoin) {
		throw new IllegalArgumentException("Bitmap filter may not be activated for joining with empty build side");
	}
	this.probeSideOuterJoin = probeSideOuterJoin;
	this.buildSideOuterJoin = buildSideOuterJoin;
	
	this.hashJoin = getHashJoin(serializer2, comparator2, serializer1,
			comparator1, pairComparator, memManager, ioManager, ownerTask, memoryFraction, useBitmapFilters);
}
 
Example #27
Source File: TupleComparatorILDXC2Test.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected TupleComparator<Tuple3<Integer, Long, Double>> createComparator(boolean ascending) {
	return new TupleComparator<Tuple3<Integer, Long, Double>>(
			new int[]{2, 1},
			new TypeComparator[]{
				new DoubleComparator(ascending),
				new LongComparator(ascending)
			},
	new TypeSerializer[]{ IntSerializer.INSTANCE, DoubleSerializer.INSTANCE, LongSerializer.INSTANCE });
}
 
Example #28
Source File: PojoComparator.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void getFlatComparator(List<TypeComparator> flatComparators) {
	for(int i = 0; i < comparators.length; i++) {
		if(comparators[i] instanceof CompositeTypeComparator) {
			((CompositeTypeComparator)comparators[i]).getFlatComparator(flatComparators);
		} else {
			flatComparators.add(comparators[i]);
		}
	}
}
 
Example #29
Source File: LeftOuterJoinDriver.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
		DriverStrategy driverStrategy,
		MutableObjectIterator<IT1> in1,
		MutableObjectIterator<IT2> in2,
		TypeSerializer<IT1> serializer1,
		TypeComparator<IT1> comparator1,
		TypeSerializer<IT2> serializer2,
		TypeComparator<IT2> comparator2,
		TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
		MemoryManager memoryManager,
		IOManager ioManager,
		double driverMemFraction
) throws Exception {
	switch (driverStrategy) {
		case LEFT_OUTER_MERGE:
			int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
			return new NonReusingMergeOuterJoinIterator<>(
					OuterJoinType.LEFT,
					in1,
					in2,
					serializer1,
					comparator1,
					serializer2,
					comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					memoryManager,
					ioManager,
					numPages,
					super.taskContext.getContainingTask()
			);
		case LEFT_HYBRIDHASH_BUILD_FIRST:
			return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator21(comparator1, comparator2),
					memoryManager, ioManager,
					this.taskContext.getContainingTask(),
					driverMemFraction,
					false,
					true,
					false);
		case LEFT_HYBRIDHASH_BUILD_SECOND:
			return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
					serializer1, comparator1,
					serializer2, comparator2,
					pairComparatorFactory.createComparator12(comparator1, comparator2),
					memoryManager, ioManager,
					this.taskContext.getContainingTask(),
					driverMemFraction,
					true,
					false,
					false);
		default:
			throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
	}
}
 
Example #30
Source File: TestData.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public static TypeComparator<Tuple2<Integer, String>> getIntStringTupleComparator() {
	return getIntStringTupleTypeInfo().createComparator(new int[]{0}, new boolean[]{true}, 0, null);
}