Java Code Examples for org.apache.flink.runtime.memory.MemoryManager

The following examples show how to use org.apache.flink.runtime.memory.MemoryManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: TestRuntimeContext.java    License: Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example 2
Source Project: flink   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 3
Source Project: flink   Source File: LargeRecordHandler.java    License: Apache License 2.0 6 votes vote down vote up
public LargeRecordHandler(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		IOManager ioManager, MemoryManager memManager, List<MemorySegment> memory,
		AbstractInvokable memoryOwner, int maxFilehandles)
{
	this.serializer = checkNotNull(serializer);
	this.comparator = checkNotNull(comparator);
	this.ioManager = checkNotNull(ioManager);
	this.memManager = checkNotNull(memManager);
	this.memory = checkNotNull(memory);
	this.memoryOwner = checkNotNull(memoryOwner);
	this.maxFilehandles = maxFilehandles;

	this.executionConfig = memoryOwner.getExecutionConfig();

	checkArgument(maxFilehandles >= 2);
}
 
Example 4
Source Project: flink   Source File: AbstractMergeOuterJoinIterator.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.outerJoinType = outerJoinType;
}
 
Example 5
private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 
Example 6
Source Project: flink   Source File: BinaryHashTableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSparseProbeSpillingWithOuterJoin() throws IOException {
	final int numBuildKeys = 1000000;
	final int numBuildVals = 1;
	final int numProbeKeys = 20;
	final int numProbeVals = 1;

	MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(
			numBuildKeys, numBuildVals, false);
	MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(96 * PAGE_SIZE).build();
	final BinaryHashTable table = new BinaryHashTable(conf, new Object(),
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(), memManager, 96 * PAGE_SIZE,
			ioManager, 24, 200000, true, HashJoinType.BUILD_OUTER, null, true, new boolean[] {true}, false);

	int expectedNumResults =
			(Math.max(numProbeKeys, numBuildKeys) * numBuildVals) * numProbeVals;

	int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true), true);

	Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);

	table.close();
	table.free();
}
 
Example 7
Source Project: Flink-CEPplus   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 8
Source Project: flink   Source File: TaskManagerServicesBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public TaskManagerServicesBuilder() {
	taskManagerLocation = new LocalTaskManagerLocation();
	memoryManager = new MemoryManager(
		MemoryManager.MIN_PAGE_SIZE,
		1,
		MemoryManager.MIN_PAGE_SIZE,
		MemoryType.HEAP,
		false);
	ioManager = mock(IOManager.class);
	shuffleEnvironment = mock(ShuffleEnvironment.class);
	kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	broadcastVariableManager = new BroadcastVariableManager();
	taskEventDispatcher = new TaskEventDispatcher();
	taskSlotTable = mock(TaskSlotTable.class);
	jobManagerTable = new JobManagerTable();
	jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
	taskStateManager = mock(TaskExecutorLocalStateStoresManager.class);
}
 
Example 9
Source Project: flink   Source File: ReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 10
Source Project: flink   Source File: TestRuntimeContext.java    License: Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example 11
Source Project: flink   Source File: ConfigurationParserUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses the configuration to get the page size and validates the value.
 *
 * @param configuration configuration object
 * @return size of memory segment
 */
public static int getPageSize(Configuration configuration) {
	final int pageSize = checkedDownCast(
		configuration.get(TaskManagerOptions.MEMORY_SEGMENT_SIZE).getBytes());

	// check page size of for minimum size
	checkConfigParameter(
		pageSize >= MemoryManager.MIN_PAGE_SIZE,
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
	// check page size for power of two
	checkConfigParameter(
		MathUtils.isPowerOf2(pageSize),
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Memory segment size must be a power of 2.");

	return pageSize;
}
 
Example 12
Source Project: Flink-CEPplus   Source File: ReOpenableHashTableTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example 13
Source Project: flink   Source File: HashJoinIteratorBase.java    License: Apache License 2.0 6 votes vote down vote up
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer,
		TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBloomFilters) throws MemoryAllocationException {

	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new MutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager,
			useBloomFilters);
}
 
Example 14
Source Project: flink   Source File: BufferDataOverWindowOperator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open() throws Exception {
	super.open();

	ClassLoader cl = getUserCodeClassloader();
	serializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(cl);
	partitionComparator = genComparator.newInstance(cl);
	genComparator = null;

	MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
	LazyMemorySegmentPool pool = new LazyMemorySegmentPool(
			this,
			memManager,
			(int) (computeMemorySize() / memManager.getPageSize()));
	this.currentData = new ResettableExternalBuffer(
			getContainingTask().getEnvironment().getIOManager(),
			pool,
			serializer, isRowAllInFixedPart);

	collector = new StreamRecordCollector<>(output);
	joinedRows = new JoinedRowData[overWindowFrames.length];
	for (int i = 0; i < overWindowFrames.length; i++) {
		overWindowFrames[i].open(new ExecutionContextImpl(this, getRuntimeContext()));
		joinedRows[i] = new JoinedRowData();
	}
}
 
Example 15
Source Project: flink   Source File: LargeRecordHandler.java    License: Apache License 2.0 6 votes vote down vote up
public LargeRecordHandler(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		IOManager ioManager, MemoryManager memManager, List<MemorySegment> memory,
		AbstractInvokable memoryOwner, int maxFilehandles)
{
	this.serializer = checkNotNull(serializer);
	this.comparator = checkNotNull(comparator);
	this.ioManager = checkNotNull(ioManager);
	this.memManager = checkNotNull(memManager);
	this.memory = checkNotNull(memory);
	this.memoryOwner = checkNotNull(memoryOwner);
	this.maxFilehandles = maxFilehandles;

	this.executionConfig = memoryOwner.getExecutionConfig();

	checkArgument(maxFilehandles >= 2);
}
 
Example 16
public ReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 
Example 17
Source Project: flink   Source File: TestRuntimeContext.java    License: Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example 18
Source Project: flink   Source File: UnilateralSortMerger.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example 19
Source Project: flink   Source File: AbstractBlockResettableIterator.java    License: Apache License 2.0 6 votes vote down vote up
protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 
Example 20
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example 21
Source Project: flink   Source File: AbstractPythonFunctionOperator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reserves the memory used by the Python worker from the MemoryManager. This makes sure that
 * the memory used by the Python worker is managed by Flink.
 */
private void reserveMemoryForPythonWorker() throws MemoryReservationException {
	long requiredPythonWorkerMemory = MemorySize.parse(config.getPythonFrameworkMemorySize())
		.add(MemorySize.parse(config.getPythonDataBufferMemorySize()))
		.getBytes();
	MemoryManager memoryManager = getContainingTask().getEnvironment().getMemoryManager();
	long availableManagedMemory = memoryManager.computeMemorySize(
		getOperatorConfig().getManagedMemoryFraction());
	if (requiredPythonWorkerMemory <= availableManagedMemory) {
		memoryManager.reserveMemory(this, requiredPythonWorkerMemory);
		LOG.info("Reserved memory {} for Python worker.", requiredPythonWorkerMemory);
		this.reservedMemory = requiredPythonWorkerMemory;
		// TODO enforce the memory limit of the Python worker
	} else {
		LOG.warn("Required Python worker memory {} exceeds the available managed off-heap " +
				"memory {}. Skipping reserving off-heap memory from the MemoryManager. This does " +
				"not affect the functionality. However, it may affect the stability of a job as " +
				"the memory used by the Python worker is not managed by Flink.",
			requiredPythonWorkerMemory, availableManagedMemory);
		this.reservedMemory = -1;
	}
}
 
Example 22
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example 23
Source Project: flink   Source File: BinaryHashTableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void validateSpillingDuringInsertion() throws IOException, MemoryAllocationException {
	final int numBuildKeys = 500000;
	final int numBuildVals = 1;
	final int numProbeKeys = 10;
	final int numProbeVals = 1;

	MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numBuildKeys, numBuildVals, false);
	MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(85 * PAGE_SIZE).build();
	final BinaryHashTable table = newBinaryHashTable(
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(), memManager,
			85 * PAGE_SIZE, ioManager);

	int expectedNumResults = (Math.min(numProbeKeys, numBuildKeys) * numBuildVals)
			* numProbeVals;

	int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true));

	Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);

	table.close();

	table.free();
}
 
Example 24
Source Project: flink   Source File: CollectSinkFunctionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	ioManager = new IOManagerAsync();
	MockEnvironment environment = new MockEnvironmentBuilder()
		.setTaskName("mockTask")
		.setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
		.setIOManager(ioManager)
		.build();
	runtimeContext = new MockStreamingRuntimeContext(false, 1, 0, environment);

	gateway = new MockOperatorEventGateway();
	coordinator = new CollectSinkOperatorCoordinator(SOCKET_TIMEOUT_MILLIS);
	coordinator.start();

	// only used in checkpointed tests
	functionInitializationContext = new MockFunctionInitializationContext();

	jobFinished = false;
}
 
Example 25
Source Project: flink   Source File: TaskManagerServicesBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public TaskManagerServices build() {
	return new TaskManagerServices(
		unresolvedTaskManagerLocation,
		MemoryManager.MIN_PAGE_SIZE,
		ioManager,
		shuffleEnvironment,
		kvStateService,
		broadcastVariableManager,
		taskSlotTable,
		jobTable,
		jobLeaderService,
		taskStateManager,
		taskEventDispatcher,
		ioExecutor,
		libraryCacheManager);
}
 
Example 26
@Before
public void beforeTest() {
	ExecutionConfig config = new ExecutionConfig();
	config.disableObjectReuse();
	
	TupleTypeInfo<Tuple2<String, String>> typeInfo1 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, String.class);
	TupleTypeInfo<Tuple2<String, Integer>> typeInfo2 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class);
	serializer1 = typeInfo1.createSerializer(config);
	serializer2 = typeInfo2.createSerializer(config);
	comparator1 = typeInfo1.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	comparator2 = typeInfo2.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	pairComp = new GenericPairComparator<>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 27
Source Project: flink   Source File: ReusingMergeInnerJoinIterator.java    License: Apache License 2.0 6 votes vote down vote up
public ReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example 28
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example 29
Source Project: flink   Source File: AbstractMergeIterator.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2,
							TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
							TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
							TypePairComparator<T1, T2> pairComparator,
							MemoryManager memoryManager,
							IOManager ioManager,
							int numMemoryPages,
							AbstractInvokable parentTask) throws MemoryAllocationException {
	if (numMemoryPages < 2) {
		throw new IllegalArgumentException("Merger needs at least 2 memory pages.");
	}

	this.pairComparator = pairComparator;
	this.serializer1 = serializer1;
	this.serializer2 = serializer2;

	this.memoryManager = memoryManager;
	this.ioManager = ioManager;

	this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate());
	this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate());

	final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1;
	this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2,
			(numMemoryPages - numPagesForSpiller), parentTask);
	this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller);
}
 
Example 30
public ReusingBlockResettableIterator(MemoryManager memoryManager, Iterator<T> input,
		TypeSerializer<T> serializer, int numPages,
		AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	this(memoryManager, serializer, numPages, ownerTask);
	this.input = input;
}