Java Code Examples for org.apache.flink.api.common.typeutils.TypeComparator#getFlatComparators()

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparator#getFlatComparators() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 
Example 2
Source Project: flink   File: RangeBoundaryBuilder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 
Example 3
Source Project: flink   File: RangeBoundaryBuilder.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void mapPartition(Iterable<T> values, Collector<Object[][]> out) throws Exception {
	final TypeComparator<T> comparator = this.comparatorFactory.createComparator();
	List<T> sampledData = new ArrayList<>();
	for (T value : values) {
		sampledData.add(value);
	}
	Collections.sort(sampledData, new Comparator<T>() {
		@Override
		public int compare(T first, T second) {
			return comparator.compare(first, second);
		}
	});


	int boundarySize = parallelism - 1;
	Object[][] boundaries = new Object[boundarySize][];
	if (sampledData.size() > 0) {
		double avgRange = sampledData.size() / (double) parallelism;
		int numKey = comparator.getFlatComparators().length;
		for (int i = 1; i < parallelism; i++) {
			T record = sampledData.get((int) (i * avgRange));
			Object[] keys = new Object[numKey];
			comparator.extractKeys(record, keys, 0);
			boundaries[i-1] = keys;
		}
	}

	out.collect(boundaries);
}
 
Example 4
public NullAwareComparator(TypeComparator<T> wrappedComparator, boolean order) {
	this.wrappedComparator = wrappedComparator;
	this.order = order;
	this.flatFields = wrappedComparator.getFlatComparators().length;
}
 
Example 5
public CommonRangeBoundaries(TypeComparator<T> typeComparators, Object[][] boundaries) {
	this.typeComparator = typeComparators;
	this.flatComparators = typeComparators.getFlatComparators();
	this.keys = new Object[flatComparators.length];
	this.boundaries = boundaries;
}
 
Example 6
Source Project: Flink-CEPplus   File: OutputEmitter.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public OutputEmitter(
		ShipStrategyType strategy,
		int indexInSubtaskGroup,
		TypeComparator<T> comparator,
		Partitioner<?> partitioner,
		DataDistribution distribution) {
	if (strategy == null) { 
		throw new NullPointerException();
	}

	this.strategy = strategy;
	this.nextChannelToSendTo = indexInSubtaskGroup;
	this.comparator = comparator;
	this.partitioner = (Partitioner<Object>) partitioner;
	this.distribution = distribution;

	switch (strategy) {
	case PARTITION_CUSTOM:
		extractedKeys = new Object[1];
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANDOM:
	case PARTITION_FORCED_REBALANCE:
		break;
	case PARTITION_RANGE:
		if (comparator != null) {
			this.flatComparators = comparator.getFlatComparators();
			this.keys = new Object[flatComparators.length];
		}
		break;
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}

	if (strategy == ShipStrategyType.PARTITION_CUSTOM && partitioner == null) {
		throw new NullPointerException("Partitioner must not be null when the ship strategy is set to custom partitioning.");
	}
}
 
Example 7
Source Project: flink   File: NullAwareComparator.java    License: Apache License 2.0 4 votes vote down vote up
public NullAwareComparator(TypeComparator<T> wrappedComparator, boolean order) {
	this.wrappedComparator = wrappedComparator;
	this.order = order;
	this.flatFields = wrappedComparator.getFlatComparators().length;
}
 
Example 8
Source Project: flink   File: CommonRangeBoundaries.java    License: Apache License 2.0 4 votes vote down vote up
public CommonRangeBoundaries(TypeComparator<T> typeComparators, Object[][] boundaries) {
	this.typeComparator = typeComparators;
	this.flatComparators = typeComparators.getFlatComparators();
	this.keys = new Object[flatComparators.length];
	this.boundaries = boundaries;
}
 
Example 9
Source Project: flink   File: OutputEmitter.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public OutputEmitter(
		ShipStrategyType strategy,
		int indexInSubtaskGroup,
		TypeComparator<T> comparator,
		Partitioner<?> partitioner,
		DataDistribution distribution) {
	if (strategy == null) { 
		throw new NullPointerException();
	}

	this.strategy = strategy;
	this.nextChannelToSendTo = indexInSubtaskGroup;
	this.comparator = comparator;
	this.partitioner = (Partitioner<Object>) partitioner;
	this.distribution = distribution;

	switch (strategy) {
	case PARTITION_CUSTOM:
		extractedKeys = new Object[1];
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANDOM:
	case PARTITION_FORCED_REBALANCE:
		break;
	case PARTITION_RANGE:
		if (comparator != null) {
			this.flatComparators = comparator.getFlatComparators();
			this.keys = new Object[flatComparators.length];
		}
		break;
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}

	if (strategy == ShipStrategyType.PARTITION_CUSTOM && partitioner == null) {
		throw new NullPointerException("Partitioner must not be null when the ship strategy is set to custom partitioning.");
	}
}
 
Example 10
Source Project: flink   File: NullAwareComparator.java    License: Apache License 2.0 4 votes vote down vote up
public NullAwareComparator(TypeComparator<T> wrappedComparator, boolean order) {
	this.wrappedComparator = wrappedComparator;
	this.order = order;
	this.flatFields = wrappedComparator.getFlatComparators().length;
}
 
Example 11
Source Project: flink   File: CommonRangeBoundaries.java    License: Apache License 2.0 4 votes vote down vote up
public CommonRangeBoundaries(TypeComparator<T> typeComparators, Object[][] boundaries) {
	this.typeComparator = typeComparators;
	this.flatComparators = typeComparators.getFlatComparators();
	this.keys = new Object[flatComparators.length];
	this.boundaries = boundaries;
}
 
Example 12
Source Project: flink   File: OutputEmitter.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public OutputEmitter(
		ShipStrategyType strategy,
		int indexInSubtaskGroup,
		TypeComparator<T> comparator,
		Partitioner<?> partitioner,
		DataDistribution distribution) {
	if (strategy == null) { 
		throw new NullPointerException();
	}

	this.strategy = strategy;
	this.nextChannelToSendTo = indexInSubtaskGroup;
	this.comparator = comparator;
	this.partitioner = (Partitioner<Object>) partitioner;
	this.distribution = distribution;

	switch (strategy) {
	case PARTITION_CUSTOM:
		extractedKeys = new Object[1];
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANDOM:
	case PARTITION_FORCED_REBALANCE:
		break;
	case PARTITION_RANGE:
		if (comparator != null) {
			this.flatComparators = comparator.getFlatComparators();
			this.keys = new Object[flatComparators.length];
		}
		break;
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}

	if (strategy == ShipStrategyType.PARTITION_CUSTOM && partitioner == null) {
		throw new NullPointerException("Partitioner must not be null when the ship strategy is set to custom partitioning.");
	}
}