org.apache.flink.api.common.typeutils.TypeSerializerFactory Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.TypeSerializerFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AllReduceDriver.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #2
Source File: UnilateralSortMerger.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 
Example #3
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 
Example #4
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 6 votes vote down vote up
protected UnilateralSortMerger(MemoryManager memoryManager, List<MemorySegment> memory,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled) throws IOException {
	this (
		memoryManager,
		memory,
		ioManager,
		input,
		parentTask,
		serializerFactory,
		comparator,
		numSortBuffers,
		maxNumFileHandles,
		startSpillingFraction,
		noSpillingMemory,
		handleLargeRecords,
		objectReuseEnabled,
		new DefaultInMemorySorterFactory<>(serializerFactory, comparator, THRESHOLD_FOR_IN_PLACE_SORTING));
}
 
Example #5
Source File: GroupReduceCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 
Example #6
Source File: TaskConfig.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void setTypeSerializerFactory(TypeSerializerFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 
Example #7
Source File: ChainedAllReduceDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final ReduceFunction<IT> red = BatchTask.instantiateUserCode(this.config, userCodeClassLoader, ReduceFunction.class);
	this.reducer = red;
	FunctionUtils.setFunctionRuntimeContext(red, getUdfRuntimeContext());

	TypeSerializerFactory<IT> serializerFactory = this.config.getInputSerializer(0, userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ChainedAllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example #8
Source File: GroupReduceCombineDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 
Example #9
Source File: UnilateralSortMerger.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new SpillingThread(exceptionHandler, queues, parentTask,
		memoryManager, ioManager, serializerFactory.getSerializer(), comparator, sortReadMemory, writeMemory, maxFileHandles);
}
 
Example #10
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index < 0 || index >= this.driver.getNumberOfInputs()) {
		throw new IndexOutOfBoundsException();
	}

	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory = (TypeSerializerFactory<X>) this.inputSerializers[index];
	return serializerFactory;
}
 
Example #11
Source File: TestTaskContext.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	switch (index) {
	case 0:
		return (TypeSerializerFactory<X>) this.serializer1;
	case 1:
		return (TypeSerializerFactory<X>) this.serializer2;
	default:
		throw new RuntimeException();
	}
}
 
Example #12
Source File: UnaryOperatorTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index != 0) {
		throw new IllegalArgumentException();
	}
	
	@SuppressWarnings("unchecked")
	TypeSerializer<X> ser = (TypeSerializer<X>) inputSerializer;
	return new RuntimeSerializerFactory<X>(ser, (Class<X>) ser.createInstance().getClass());
}
 
Example #13
Source File: CombiningUnilateralSortMerger.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 
Example #14
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates all the serializers and iterators for the broadcast inputs.
 */
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
	this.broadcastInputSerializers = new TypeSerializerFactory<?>[numBroadcastInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();

	for (int i = 0; i < numBroadcastInputs; i++) {
		//  ---------------- create the serializer first ---------------------
		final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, userCodeClassLoader);
		this.broadcastInputSerializers[i] = serializerFactory;
	}
}
 
Example #15
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 
Example #16
Source File: AbstractIterativeTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * @return output serializer of this task
 */
private TypeSerializer<OT> getOutputSerializer() {
	TypeSerializerFactory<OT> serializerFactory;

	if ((serializerFactory = getLastTasksConfig().getOutputSerializer(getUserCodeClassLoader())) ==
			null) {
		throw new RuntimeException("Missing output serializer for workset update.");
	}

	return serializerFactory.getSerializer();
}
 
Example #17
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 
Example #18
Source File: CombiningUnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 
Example #19
Source File: IterationHeadTask.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 
Example #20
Source File: TaskConfig.java    From flink with Apache License 2.0 5 votes vote down vote up
private void setTypeSerializerFactory(TypeSerializerFactory<?> factory,
		String classNameKey, String parametersPrefix)
{
	// sanity check the factory type
	InstantiationUtil.checkForInstantiation(factory.getClass());
	
	// store the type
	this.config.setString(classNameKey, factory.getClass().getName());
	// store the parameters
	final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix);
	factory.writeParametersToConfig(parameters);
}
 
Example #21
Source File: UnaryOperatorTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	if (index != 0) {
		throw new IllegalArgumentException();
	}
	
	@SuppressWarnings("unchecked")
	TypeSerializer<X> ser = (TypeSerializer<X>) inputSerializer;
	return new RuntimeSerializerFactory<X>(ser, (Class<X>) ser.createInstance().getClass());
}
 
Example #22
Source File: IterationHeadTask.java    From flink with Apache License 2.0 5 votes vote down vote up
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 
Example #23
Source File: TestTaskContext.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <X> TypeSerializerFactory<X> getInputSerializer(int index) {
	switch (index) {
	case 0:
		return (TypeSerializerFactory<X>) this.serializer1;
	case 1:
		return (TypeSerializerFactory<X>) this.serializer2;
	default:
		throw new RuntimeException();
	}
}
 
Example #24
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates all the serializers and iterators for the broadcast inputs.
 */
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
	this.broadcastInputSerializers = new TypeSerializerFactory<?>[numBroadcastInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();

	for (int i = 0; i < numBroadcastInputs; i++) {
		//  ---------------- create the serializer first ---------------------
		final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, userCodeClassLoader);
		this.broadcastInputSerializers[i] = serializerFactory;
	}
}
 
Example #25
Source File: BatchTask.java    From flink with Apache License 2.0 5 votes vote down vote up
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 
Example #26
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
protected UnilateralSortMerger(MemoryManager memoryManager,
		IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean noSpillingMemory, boolean handleLargeRecords,
		boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, memoryManager.allocatePages(parentTask, memoryManager.computeNumberOfPages(memoryFraction)),
			ioManager, input, parentTask, serializerFactory, comparator,
			numSortBuffers, maxNumFileHandles, startSpillingFraction, noSpillingMemory, handleLargeRecords,
			objectReuseEnabled);
}
 
Example #27
Source File: CombiningUnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
	AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
	TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
	List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new CombiningSpillingThread(exceptionHandler, queues, parentTask,
			memoryManager, ioManager, serializerFactory.getSerializer(),
			comparator, sortReadMemory, writeMemory, maxFileHandles, objectReuseEnabled);
}
 
Example #28
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int maxNumFileHandles, float startSpillingFraction,
		boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, -1, maxNumFileHandles, startSpillingFraction, handleLargeRecords, objectReuseEnabled);
}
 
Example #29
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
public UnilateralSortMerger(MemoryManager memoryManager, IOManager ioManager,
		MutableObjectIterator<E> input, AbstractInvokable parentTask,
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		double memoryFraction, int numSortBuffers, int maxNumFileHandles,
		float startSpillingFraction, boolean handleLargeRecords, boolean objectReuseEnabled)
throws IOException, MemoryAllocationException
{
	this(memoryManager, ioManager, input, parentTask, serializerFactory, comparator,
		memoryFraction, numSortBuffers, maxNumFileHandles, startSpillingFraction, false, handleLargeRecords,
		objectReuseEnabled);
}
 
Example #30
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
protected ThreadBase<E> getSpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memoryManager, IOManager ioManager, 
		TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxFileHandles)
{
	return new SpillingThread(exceptionHandler, queues, parentTask,
		memoryManager, ioManager, serializerFactory.getSerializer(), comparator, sortReadMemory, writeMemory, maxFileHandles);
}