org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator Java Examples

The following examples show how to use org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 *
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {

	final MemoryManager memMan = getMemoryManager();
	final IOManager ioMan = getIOManager();

	this.localStrategies = new CloseableInputProvider<?>[numInputs];
	this.inputs = new MutableObjectIterator<?>[numInputs];
	this.excludeFromReset = new boolean[numInputs];
	this.inputIsCached = new boolean[numInputs];
	this.inputIsAsyncMaterialized = new boolean[numInputs];
	this.materializationMemory = new int[numInputs];

	// set up the local strategies first, such that the can work before any temp barrier is created
	for (int i = 0; i < numInputs; i++) {
		initInputLocalStrategy(i);
	}

	// we do another loop over the inputs, because we want to instantiate all
	// sorters, etc before requesting the first input (as this call may block)

	// we have two types of materialized inputs, and both are replayable (can act as a cache)
	// The first variant materializes in a different thread and hence
	// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
	// the second variant spills to the side and will not read unless the result is also consumed
	// in a pipelined fashion.
	this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
	this.tempBarriers = new TempBarrier<?>[numInputs];

	for (int i = 0; i < numInputs; i++) {
		final int memoryPages;
		final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
		final boolean cached =  this.config.isInputCached(i);

		this.inputIsAsyncMaterialized[i] = async;
		this.inputIsCached[i] = cached;

		if (async || cached) {
			memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
			if (memoryPages <= 0) {
				throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
			}
			this.materializationMemory[i] = memoryPages;
		} else {
			memoryPages = 0;
		}

		if (async) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
			barrier.startReading();
			this.tempBarriers[i] = barrier;
			this.inputs[i] = null;
		} else if (cached) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
				getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
			this.resettableInputs[i] = iter;
			this.inputs[i] = iter;
		}
	}
}
Example #2
Source File: BatchTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 *
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {

	final MemoryManager memMan = getMemoryManager();
	final IOManager ioMan = getIOManager();

	this.localStrategies = new CloseableInputProvider<?>[numInputs];
	this.inputs = new MutableObjectIterator<?>[numInputs];
	this.excludeFromReset = new boolean[numInputs];
	this.inputIsCached = new boolean[numInputs];
	this.inputIsAsyncMaterialized = new boolean[numInputs];
	this.materializationMemory = new int[numInputs];

	// set up the local strategies first, such that the can work before any temp barrier is created
	for (int i = 0; i < numInputs; i++) {
		initInputLocalStrategy(i);
	}

	// we do another loop over the inputs, because we want to instantiate all
	// sorters, etc before requesting the first input (as this call may block)

	// we have two types of materialized inputs, and both are replayable (can act as a cache)
	// The first variant materializes in a different thread and hence
	// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
	// the second variant spills to the side and will not read unless the result is also consumed
	// in a pipelined fashion.
	this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
	this.tempBarriers = new TempBarrier<?>[numInputs];

	for (int i = 0; i < numInputs; i++) {
		final int memoryPages;
		final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
		final boolean cached =  this.config.isInputCached(i);

		this.inputIsAsyncMaterialized[i] = async;
		this.inputIsCached[i] = cached;

		if (async || cached) {
			memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
			if (memoryPages <= 0) {
				throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
			}
			this.materializationMemory[i] = memoryPages;
		} else {
			memoryPages = 0;
		}

		if (async) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
			barrier.startReading();
			this.tempBarriers[i] = barrier;
			this.inputs[i] = null;
		} else if (cached) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
				getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
			this.resettableInputs[i] = iter;
			this.inputs[i] = iter;
		}
	}
}
Example #3
Source File: BatchTask.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 *
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {

	final MemoryManager memMan = getMemoryManager();
	final IOManager ioMan = getIOManager();

	this.localStrategies = new CloseableInputProvider<?>[numInputs];
	this.inputs = new MutableObjectIterator<?>[numInputs];
	this.excludeFromReset = new boolean[numInputs];
	this.inputIsCached = new boolean[numInputs];
	this.inputIsAsyncMaterialized = new boolean[numInputs];
	this.materializationMemory = new int[numInputs];

	// set up the local strategies first, such that the can work before any temp barrier is created
	for (int i = 0; i < numInputs; i++) {
		initInputLocalStrategy(i);
	}

	// we do another loop over the inputs, because we want to instantiate all
	// sorters, etc before requesting the first input (as this call may block)

	// we have two types of materialized inputs, and both are replayable (can act as a cache)
	// The first variant materializes in a different thread and hence
	// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
	// the second variant spills to the side and will not read unless the result is also consumed
	// in a pipelined fashion.
	this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
	this.tempBarriers = new TempBarrier<?>[numInputs];

	for (int i = 0; i < numInputs; i++) {
		final int memoryPages;
		final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
		final boolean cached =  this.config.isInputCached(i);

		this.inputIsAsyncMaterialized[i] = async;
		this.inputIsCached[i] = cached;

		if (async || cached) {
			memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
			if (memoryPages <= 0) {
				throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
			}
			this.materializationMemory[i] = memoryPages;
		} else {
			memoryPages = 0;
		}

		if (async) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
			barrier.startReading();
			this.tempBarriers[i] = barrier;
			this.inputs[i] = null;
		} else if (cached) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
				getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
			this.resettableInputs[i] = iter;
			this.inputs[i] = iter;
		}
	}
}