org.apache.flink.runtime.io.network.api.reader.MutableReader Java Examples

The following examples show how to use org.apache.flink.runtime.io.network.api.reader.MutableReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 
Example #9
Source Project: flink   Author: apache   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 
Example #10
Source Project: flink   Author: apache   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 
Example #11
Source Project: flink   Author: apache   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 
Example #12
Source Project: flink   Author: apache   File: BatchTask.java    License: Apache License 2.0 5 votes vote down vote up
protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: BroadcastVariableManager.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 
Example #14
Source Project: Flink-CEPplus   Author: ljygz   File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: ReaderIterator.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: BroadcastVariableManager.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: ReaderIterator.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}
 
Example #19
Source Project: flink   Author: apache   File: BroadcastVariableManager.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 
Example #20
Source Project: flink   Author: apache   File: BatchTask.java    License: Apache License 2.0 4 votes vote down vote up
public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 
Example #21
Source Project: flink   Author: apache   File: ReaderIterator.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}