org.apache.flink.runtime.memory.MemoryManager Java Examples

The following examples show how to use org.apache.flink.runtime.memory.MemoryManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRuntimeContext.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example #2
Source File: UnaryOperatorTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example #3
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 6 votes vote down vote up
public LargeRecordHandler(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		IOManager ioManager, MemoryManager memManager, List<MemorySegment> memory,
		AbstractInvokable memoryOwner, int maxFilehandles)
{
	this.serializer = checkNotNull(serializer);
	this.comparator = checkNotNull(comparator);
	this.ioManager = checkNotNull(ioManager);
	this.memManager = checkNotNull(memManager);
	this.memory = checkNotNull(memory);
	this.memoryOwner = checkNotNull(memoryOwner);
	this.maxFilehandles = maxFilehandles;

	this.executionConfig = memoryOwner.getExecutionConfig();

	checkArgument(maxFilehandles >= 2);
}
 
Example #4
Source File: AbstractMergeOuterJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
public AbstractMergeOuterJoinIterator(
		OuterJoinType outerJoinType,
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.outerJoinType = outerJoinType;
}
 
Example #5
Source File: SpillingResettableMutableObjectIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
private SpillingResettableMutableObjectIterator(MutableObjectIterator<T> input, TypeSerializer<T> serializer,
		MemoryManager memoryManager, IOManager ioManager,
		List<MemorySegment> memory, boolean releaseMemOnClose)
{
	this.memoryManager = memoryManager;
	this.input = input;
	this.serializer = serializer;
	this.memorySegments = memory;
	this.releaseMemoryOnClose = releaseMemOnClose;
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
	}
	
	this.buffer = new SpillingBuffer(ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
 
Example #6
Source File: BinaryHashTableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSparseProbeSpillingWithOuterJoin() throws IOException {
	final int numBuildKeys = 1000000;
	final int numBuildVals = 1;
	final int numProbeKeys = 20;
	final int numProbeVals = 1;

	MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(
			numBuildKeys, numBuildVals, false);
	MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(96 * PAGE_SIZE).build();
	final BinaryHashTable table = new BinaryHashTable(conf, new Object(),
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(), memManager, 96 * PAGE_SIZE,
			ioManager, 24, 200000, true, HashJoinType.BUILD_OUTER, null, true, new boolean[] {true}, false);

	int expectedNumResults =
			(Math.max(numProbeKeys, numBuildKeys) * numBuildVals) * numProbeVals;

	int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true), true);

	Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);

	table.close();
	table.free();
}
 
Example #7
Source File: DriverTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example #8
Source File: TaskManagerServicesBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskManagerServicesBuilder() {
	taskManagerLocation = new LocalTaskManagerLocation();
	memoryManager = new MemoryManager(
		MemoryManager.MIN_PAGE_SIZE,
		1,
		MemoryManager.MIN_PAGE_SIZE,
		MemoryType.HEAP,
		false);
	ioManager = mock(IOManager.class);
	shuffleEnvironment = mock(ShuffleEnvironment.class);
	kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	broadcastVariableManager = new BroadcastVariableManager();
	taskEventDispatcher = new TaskEventDispatcher();
	taskSlotTable = mock(TaskSlotTable.class);
	jobManagerTable = new JobManagerTable();
	jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
	taskStateManager = mock(TaskExecutorLocalStateStoresManager.class);
}
 
Example #9
Source File: ReusingHashJoinIteratorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example #10
Source File: TestRuntimeContext.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example #11
Source File: ConfigurationParserUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Parses the configuration to get the page size and validates the value.
 *
 * @param configuration configuration object
 * @return size of memory segment
 */
public static int getPageSize(Configuration configuration) {
	final int pageSize = checkedDownCast(
		configuration.get(TaskManagerOptions.MEMORY_SEGMENT_SIZE).getBytes());

	// check page size of for minimum size
	checkConfigParameter(
		pageSize >= MemoryManager.MIN_PAGE_SIZE,
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
	// check page size for power of two
	checkConfigParameter(
		MathUtils.isPowerOf2(pageSize),
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Memory segment size must be a power of 2.");

	return pageSize;
}
 
Example #12
Source File: ReOpenableHashTableTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example #13
Source File: HashJoinIteratorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer,
		TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBloomFilters) throws MemoryAllocationException {

	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new MutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager,
			useBloomFilters);
}
 
Example #14
Source File: BufferDataOverWindowOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open() throws Exception {
	super.open();

	ClassLoader cl = getUserCodeClassloader();
	serializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(cl);
	partitionComparator = genComparator.newInstance(cl);
	genComparator = null;

	MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
	LazyMemorySegmentPool pool = new LazyMemorySegmentPool(
			this,
			memManager,
			(int) (computeMemorySize() / memManager.getPageSize()));
	this.currentData = new ResettableExternalBuffer(
			getContainingTask().getEnvironment().getIOManager(),
			pool,
			serializer, isRowAllInFixedPart);

	collector = new StreamRecordCollector<>(output);
	joinedRows = new JoinedRowData[overWindowFrames.length];
	for (int i = 0; i < overWindowFrames.length; i++) {
		overWindowFrames[i].open(new ExecutionContextImpl(this, getRuntimeContext()));
		joinedRows[i] = new JoinedRowData();
	}
}
 
Example #15
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 6 votes vote down vote up
public LargeRecordHandler(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		IOManager ioManager, MemoryManager memManager, List<MemorySegment> memory,
		AbstractInvokable memoryOwner, int maxFilehandles)
{
	this.serializer = checkNotNull(serializer);
	this.comparator = checkNotNull(comparator);
	this.ioManager = checkNotNull(ioManager);
	this.memManager = checkNotNull(memManager);
	this.memory = checkNotNull(memory);
	this.memoryOwner = checkNotNull(memoryOwner);
	this.maxFilehandles = maxFilehandles;

	this.executionConfig = memoryOwner.getExecutionConfig();

	checkArgument(maxFilehandles >= 2);
}
 
Example #16
Source File: ReusingBuildSecondReOpenableHashJoinIterator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public ReusingBuildSecondReOpenableHashJoinIterator(
		MutableObjectIterator<V1> firstInput,
		MutableObjectIterator<V2> secondInput,
		TypeSerializer<V1> serializer1,
		TypeComparator<V1> comparator1,
		TypeSerializer<V2> serializer2,
		TypeComparator<V2> comparator2,
		TypePairComparator<V1, V2> pairComparator,
		MemoryManager memManager,
		IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean probeSideOuterJoin,
		boolean buildSideOuterJoin,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	super(firstInput, secondInput, serializer1, comparator1, serializer2,
			comparator2, pairComparator, memManager, ioManager, ownerTask,
			memoryFraction, probeSideOuterJoin, buildSideOuterJoin, useBitmapFilters);
	
	reopenHashTable = (ReOpenableMutableHashTable<V2, V1>) hashJoin;
}
 
Example #17
Source File: TestRuntimeContext.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestRuntimeContext(
	boolean isCheckpointingEnabled,
	int numParallelSubtasks,
	int subtaskIndex) {

	super(
		new TestStreamOperator(),
		new MockEnvironmentBuilder()
			.setTaskName("mockTask")
			.setMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
			.build(),
		Collections.emptyMap());

	this.isCheckpointingEnabled = isCheckpointingEnabled;
	this.numParallelSubtasks = numParallelSubtasks;
	this.subtaskIndex = subtaskIndex;
}
 
Example #18
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the spilling thread.
 * 
 * @param exceptionHandler The exception handler to call for all exceptions.
 * @param queues The queues used to pass buffers between the threads.
 * @param parentTask The task that started this thread. If non-null, it is used to register this thread.
 * @param memManager The memory manager used to allocate buffers for the readers and writers.
 * @param ioManager The I/I manager used to instantiate readers and writers from.
 * @param serializer
 * @param comparator
 * @param sortReadMemory
 * @param writeMemory
 * @param maxNumFileHandles
 */
public SpillingThread(ExceptionHandler<IOException> exceptionHandler, CircularQueues<E> queues,
		AbstractInvokable parentTask, MemoryManager memManager, IOManager ioManager, 
		TypeSerializer<E> serializer, TypeComparator<E> comparator, 
		List<MemorySegment> sortReadMemory, List<MemorySegment> writeMemory, int maxNumFileHandles)
{
	super(exceptionHandler, "SortMerger spilling thread", queues, parentTask);
	this.memManager = memManager;
	this.ioManager = ioManager;
	this.serializer = serializer;
	this.comparator = comparator;
	this.mergeReadMemory = sortReadMemory;
	this.writeMemory = writeMemory;
	this.maxFanIn = maxNumFileHandles;
	this.numWriteBuffersToCluster = writeMemory.size() >= 4 ? writeMemory.size() / 2 : 1;
}
 
Example #19
Source File: AbstractBlockResettableIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
protected AbstractBlockResettableIterator(TypeSerializer<T> serializer, MemoryManager memoryManager,
		int numPages, AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	if (numPages < 1) {
		throw new IllegalArgumentException("Block Resettable iterator requires at leat one page of memory");
	}
	
	this.memoryManager = memoryManager;
	this.serializer = serializer;
	
	this.emptySegments = new ArrayList<MemorySegment>(numPages);
	this.fullSegments = new ArrayList<MemorySegment>(numPages);
	memoryManager.allocatePages(ownerTask, emptySegments, numPages);
	
	this.collectingView = new SimpleCollectingOutputView(this.fullSegments, 
					new ListMemorySegmentSource(this.emptySegments), memoryManager.getPageSize());
	this.readView = new RandomAccessInputView(this.fullSegments, memoryManager.getPageSize());
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Iterator initialized using " + numPages + " memory buffers.");
	}
}
 
Example #20
Source File: DriverTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.close();

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 
Example #21
Source File: AbstractPythonFunctionOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Reserves the memory used by the Python worker from the MemoryManager. This makes sure that
 * the memory used by the Python worker is managed by Flink.
 */
private void reserveMemoryForPythonWorker() throws MemoryReservationException {
	long requiredPythonWorkerMemory = MemorySize.parse(config.getPythonFrameworkMemorySize())
		.add(MemorySize.parse(config.getPythonDataBufferMemorySize()))
		.getBytes();
	MemoryManager memoryManager = getContainingTask().getEnvironment().getMemoryManager();
	long availableManagedMemory = memoryManager.computeMemorySize(
		getOperatorConfig().getManagedMemoryFraction());
	if (requiredPythonWorkerMemory <= availableManagedMemory) {
		memoryManager.reserveMemory(this, requiredPythonWorkerMemory);
		LOG.info("Reserved memory {} for Python worker.", requiredPythonWorkerMemory);
		this.reservedMemory = requiredPythonWorkerMemory;
		// TODO enforce the memory limit of the Python worker
	} else {
		LOG.warn("Required Python worker memory {} exceeds the available managed off-heap " +
				"memory {}. Skipping reserving off-heap memory from the MemoryManager. This does " +
				"not affect the functionality. However, it may affect the stability of a job as " +
				"the memory used by the Python worker is not managed by Flink.",
			requiredPythonWorkerMemory, availableManagedMemory);
		this.reservedMemory = -1;
	}
}
 
Example #22
Source File: NonReusingBuildSecondReOpenableHashJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer,
		TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example #23
Source File: BinaryHashTableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void validateSpillingDuringInsertion() throws IOException, MemoryAllocationException {
	final int numBuildKeys = 500000;
	final int numBuildVals = 1;
	final int numProbeKeys = 10;
	final int numProbeVals = 1;

	MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numBuildKeys, numBuildVals, false);
	MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(85 * PAGE_SIZE).build();
	final BinaryHashTable table = newBinaryHashTable(
			this.buildSideSerializer, this.probeSideSerializer,
			new MyProjection(), new MyProjection(), memManager,
			85 * PAGE_SIZE, ioManager);

	int expectedNumResults = (Math.min(numProbeKeys, numBuildKeys) * numBuildVals)
			* numProbeVals;

	int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true));

	Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);

	table.close();

	table.free();
}
 
Example #24
Source File: CollectSinkFunctionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	ioManager = new IOManagerAsync();
	MockEnvironment environment = new MockEnvironmentBuilder()
		.setTaskName("mockTask")
		.setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
		.setIOManager(ioManager)
		.build();
	runtimeContext = new MockStreamingRuntimeContext(false, 1, 0, environment);

	gateway = new MockOperatorEventGateway();
	coordinator = new CollectSinkOperatorCoordinator(SOCKET_TIMEOUT_MILLIS);
	coordinator.start();

	// only used in checkpointed tests
	functionInitializationContext = new MockFunctionInitializationContext();

	jobFinished = false;
}
 
Example #25
Source File: TaskManagerServicesBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskManagerServices build() {
	return new TaskManagerServices(
		unresolvedTaskManagerLocation,
		MemoryManager.MIN_PAGE_SIZE,
		ioManager,
		shuffleEnvironment,
		kvStateService,
		broadcastVariableManager,
		taskSlotTable,
		jobTable,
		jobLeaderService,
		taskStateManager,
		taskEventDispatcher,
		ioExecutor,
		libraryCacheManager);
}
 
Example #26
Source File: AbstractSortMergeOuterJoinIteratorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() {
	ExecutionConfig config = new ExecutionConfig();
	config.disableObjectReuse();
	
	TupleTypeInfo<Tuple2<String, String>> typeInfo1 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, String.class);
	TupleTypeInfo<Tuple2<String, Integer>> typeInfo2 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class);
	serializer1 = typeInfo1.createSerializer(config);
	serializer2 = typeInfo2.createSerializer(config);
	comparator1 = typeInfo1.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	comparator2 = typeInfo2.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	pairComp = new GenericPairComparator<>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example #27
Source File: ReusingMergeInnerJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
public ReusingMergeInnerJoinIterator(
		MutableObjectIterator<T1> input1,
		MutableObjectIterator<T2> input2,
		TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
		TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
		TypePairComparator<T1, T2> pairComparator,
		MemoryManager memoryManager,
		IOManager ioManager,
		int numMemoryPages,
		AbstractInvokable parentTask)
		throws MemoryAllocationException {
	super(input1, input2, serializer1, comparator1, serializer2, comparator2, pairComparator, memoryManager, ioManager, numMemoryPages, parentTask);

	this.copy1 = serializer1.createInstance();
	this.spillHeadCopy = serializer1.createInstance();
	this.copy2 = serializer2.createInstance();
	this.blockHeadCopy = serializer2.createInstance();
}
 
Example #28
Source File: ReusingBuildSecondReOpenableHashJoinIterator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
		TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
		TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
		TypePairComparator<PT, BT> pairComparator,
		MemoryManager memManager, IOManager ioManager,
		AbstractInvokable ownerTask,
		double memoryFraction,
		boolean useBitmapFilters) throws MemoryAllocationException {
	
	final int numPages = memManager.computeNumberOfPages(memoryFraction);
	final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
	
	return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
			buildSideComparator, probeSideComparator, pairComparator,
			memorySegments, ioManager, useBitmapFilters);
}
 
Example #29
Source File: AbstractMergeIterator.java    From flink with Apache License 2.0 5 votes vote down vote up
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2,
							TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1,
							TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2,
							TypePairComparator<T1, T2> pairComparator,
							MemoryManager memoryManager,
							IOManager ioManager,
							int numMemoryPages,
							AbstractInvokable parentTask) throws MemoryAllocationException {
	if (numMemoryPages < 2) {
		throw new IllegalArgumentException("Merger needs at least 2 memory pages.");
	}

	this.pairComparator = pairComparator;
	this.serializer1 = serializer1;
	this.serializer2 = serializer2;

	this.memoryManager = memoryManager;
	this.ioManager = ioManager;

	this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate());
	this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate());

	final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1;
	this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2,
			(numMemoryPages - numPagesForSpiller), parentTask);
	this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller);
}
 
Example #30
Source File: ReusingBlockResettableIterator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public ReusingBlockResettableIterator(MemoryManager memoryManager, Iterator<T> input,
		TypeSerializer<T> serializer, int numPages,
		AbstractInvokable ownerTask)
throws MemoryAllocationException
{
	this(memoryManager, serializer, numPages, ownerTask);
	this.input = input;
}