Java Code Examples for org.apache.flink.runtime.operators.util.TaskConfig#getDriverStrategy()

The following examples show how to use org.apache.flink.runtime.operators.util.TaskConfig#getDriverStrategy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AllReduceDriver.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 2

Source File: CoGroupRawDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
		throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
	}

	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);

	IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
	IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();

	this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
	this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);

	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}

Example 3

Source File: AllReduceDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 4

Source File: GroupReduceDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
		throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
	}
	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 5

Source File: GroupReduceDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
		throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
	}
	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 6

Source File: CoGroupRawDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
		throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
	}

	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);

	IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
	IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();

	this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
	this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);

	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}

Example 7

Source File: ReduceDriver.java From flink with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
		throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
	}
	this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 8

Source File: ReduceDriver.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
		throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
	}
	this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 9

Source File: CoGroupRawDriver.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
		throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
	}

	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);

	IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
	IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();

	this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
	this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);

	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}

Example 10

Source File: AllGroupReduceDriver.java From flink with Apache License 2.0

5 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	this.strategy = config.getDriverStrategy();

	switch (this.strategy) {
		case ALL_GROUP_REDUCE_COMBINE:
			if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
			}
		case ALL_GROUP_REDUCE:
		case ALL_GROUP_COMBINE:
			break;
		default:
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
	}

	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 11

Source File: AllGroupReduceDriver.java From flink with Apache License 2.0

5 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	this.strategy = config.getDriverStrategy();

	switch (this.strategy) {
		case ALL_GROUP_REDUCE_COMBINE:
			if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
			}
		case ALL_GROUP_REDUCE:
		case ALL_GROUP_COMBINE:
			break;
		default:
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
	}

	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 12

Source File: AllGroupReduceDriver.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	this.strategy = config.getDriverStrategy();

	switch (this.strategy) {
		case ALL_GROUP_REDUCE_COMBINE:
			if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
			}
		case ALL_GROUP_REDUCE:
		case ALL_GROUP_COMBINE:
			break;
		default:
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
	}

	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 13

Source File: AbstractOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}

Example 14

Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}

Example 15

Source File: CoGroupDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception
{
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
		throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
	}

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
				this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for CoGroup driver");
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}

	if (objectReuseEnabled) {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	} else {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	}
	
	// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
	this.coGroupIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}

Example 16

Source File: CrossDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	final DriverStrategy ls = config.getDriverStrategy();
	
	switch (ls)
	{
	case NESTEDLOOP_BLOCKED_OUTER_FIRST:
		this.blocked = true;
		this.firstIsOuter = true;
		break;
	case NESTEDLOOP_BLOCKED_OUTER_SECOND:
		this.blocked = true;
		this.firstIsOuter = false;
		break;
	case NESTEDLOOP_STREAMED_OUTER_FIRST:
		this.blocked = false;
		this.firstIsOuter = true;
		break;
	case NESTEDLOOP_STREAMED_OUTER_SECOND:
		this.blocked = false;
		this.firstIsOuter = false;
		break;
	default:
		throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
	}
	
	this.memManager = this.taskContext.getMemoryManager();
	final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
	
	if (numPages < 2) {
		throw new RuntimeException(	"The Cross task was initialized with too little memory. " +
				"Cross requires at least 2 memory pages.");
	}
	
	// divide memory between spilling and blocking side
	if (ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST || ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND) {
		this.memPagesForSpillingSide = numPages;
		this.memPagesForBlockSide = 0;
	} else {
		if (numPages > 32) {
			this.memPagesForSpillingSide = 2;
		} else {
			this.memPagesForSpillingSide =  1;
		}
		this.memPagesForBlockSide = numPages - this.memPagesForSpillingSide;
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CrossDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 17

Source File: CrossDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	final DriverStrategy ls = config.getDriverStrategy();
	
	switch (ls)
	{
	case NESTEDLOOP_BLOCKED_OUTER_FIRST:
		this.blocked = true;
		this.firstIsOuter = true;
		break;
	case NESTEDLOOP_BLOCKED_OUTER_SECOND:
		this.blocked = true;
		this.firstIsOuter = false;
		break;
	case NESTEDLOOP_STREAMED_OUTER_FIRST:
		this.blocked = false;
		this.firstIsOuter = true;
		break;
	case NESTEDLOOP_STREAMED_OUTER_SECOND:
		this.blocked = false;
		this.firstIsOuter = false;
		break;
	default:
		throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
	}
	
	this.memManager = this.taskContext.getMemoryManager();
	final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
	
	if (numPages < 2) {
		throw new RuntimeException(	"The Cross task was initialized with too little memory. " +
				"Cross requires at least 2 memory pages.");
	}
	
	// divide memory between spilling and blocking side
	if (ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST || ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND) {
		this.memPagesForSpillingSide = numPages;
		this.memPagesForBlockSide = 0;
	} else {
		if (numPages > 32) {
			this.memPagesForSpillingSide = 2;
		} else {
			this.memPagesForSpillingSide =  1;
		}
		this.memPagesForBlockSide = numPages - this.memPagesForSpillingSide;
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CrossDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}

Example 18

Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}

Example 19

Source File: CoGroupDriver.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception
{
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
		throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
	}

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
				this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for CoGroup driver");
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}

	if (objectReuseEnabled) {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	} else {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	}
	
	// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
	this.coGroupIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}

Example 20

Source File: CoGroupDriver.java From flink with Apache License 2.0

4 votes

@Override
public void prepare() throws Exception
{
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
		throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
	}

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
				this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for CoGroup driver");
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}

	if (objectReuseEnabled) {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	} else {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	}
	
	// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
	this.coGroupIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}