Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: ReOpenableHashTableTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example 2
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 3
Source Project: Flink-CEPplus   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example 4
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 5
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 6
@Before
public void beforeTest() {
	ExecutionConfig config = new ExecutionConfig();
	config.disableObjectReuse();
	
	TupleTypeInfo<Tuple2<String, String>> typeInfo1 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, String.class);
	TupleTypeInfo<Tuple2<String, Integer>> typeInfo2 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class);
	serializer1 = typeInfo1.createSerializer(config);
	serializer2 = typeInfo2.createSerializer(config);
	comparator1 = typeInfo1.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	comparator2 = typeInfo2.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	pairComp = new GenericPairComparator<>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 7
Source Project: Flink-CEPplus   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 8
Source Project: Flink-CEPplus   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 9
Source Project: flink   Source File: SavepointEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) {
	this.jobID = new JobID();
	this.vertexID = new JobVertexID();
	this.attemptID = new ExecutionAttemptID();
	this.ctx = Preconditions.checkNotNull(ctx);
	this.configuration = Preconditions.checkNotNull(configuration);

	Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism);
	this.maxParallelism = maxParallelism;
	this.indexOfSubtask = indexOfSubtask;

	this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID);
	this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState);
	this.ioManager = new IOManagerAsync();
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID);
}
 
Example 10
Source Project: flink   Source File: ReOpenableHashTableTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example 11
Source Project: flink   Source File: ReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 12
Source Project: flink   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example 13
Source Project: flink   Source File: NonReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 14
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 15
@Before
public void beforeTest() {
	ExecutionConfig config = new ExecutionConfig();
	config.disableObjectReuse();
	
	TupleTypeInfo<Tuple2<String, String>> typeInfo1 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, String.class);
	TupleTypeInfo<Tuple2<String, Integer>> typeInfo2 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class);
	serializer1 = typeInfo1.createSerializer(config);
	serializer2 = typeInfo2.createSerializer(config);
	comparator1 = typeInfo1.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	comparator2 = typeInfo2.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	pairComp = new GenericPairComparator<>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 16
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	serializer1 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	serializer2 = new TupleSerializer<Tuple2<Integer, String>>(
			(Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class,
			new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
	comparator1 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	comparator2 =  new TupleComparator<Tuple2<Integer, String>>(
			new int[]{0},
			new TypeComparator<?>[] { new IntComparator(true) },
			new TypeSerializer<?>[] { IntSerializer.INSTANCE });
	pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 17
Source Project: flink   Source File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 18
Source Project: flink   Source File: UnaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected UnaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	this.owner = new DummyInvokable();

	Configuration config = new Configuration();
	this.taskConfig = new TaskConfig(config);

	this.executionConfig = executionConfig;
	this.comparators = new ArrayList<TypeComparator<IN>>(2);

	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 19
Source Project: flink   Source File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected BinaryOperatorTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double) perSortMemory / totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem, 1) : null;
	
	this.inputs = new ArrayList<>();
	this.comparators = new ArrayList<>();
	this.sorters = new ArrayList<>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 
Example 20
@Before
public void startup() {
	// set up IO and memory manager
	this.memman = new MemoryManager(MEMORY_CAPACITY, 1, 32 * 1024, MemoryType.HEAP, true);
	this.ioman = new IOManagerAsync();

	// create test objects
	ArrayList<IntValue> objects = new ArrayList<IntValue>(NUM_TESTRECORDS);

	for (int i = 0; i < NUM_TESTRECORDS; ++i) {
		IntValue tmp = new IntValue(i);
		objects.add(tmp);
	}
	this.reader = objects.iterator();
}
 
Example 21
Source Project: Flink-CEPplus   Source File: HashTableTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This tests the case where no additional partition buffers are used at the point when spilling
 * is triggered, testing that overflow bucket buffers are taken into account when deciding which
 * partition to spill.
 */
@Test
public void testSpillingFreesOnlyOverflowSegments() {
	final IOManager ioMan = new IOManagerAsync();
	
	final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE;
	final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class);
	final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class);
	
	@SuppressWarnings("unchecked")
	final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class);
	
	try {
		final int pageSize = 32*1024;
		final int numSegments = 34;

		List<MemorySegment> memory = getMemory(numSegments, pageSize);

		MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>(
				serializer, serializer, buildComparator, probeComparator,
				pairComparator, memory, ioMan, 1, false);

		table.open(new ByteValueIterator(100000000), new ByteValueIterator(1));
		
		table.close();
		
		checkNoTempFilesRemain(ioMan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioMan.shutdown();
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: ReOpenableHashTableITCase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ExternalSortITCase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
	
	this.pactRecordSerializer = TestData.getIntStringTupleSerializerFactory();
	this.pactRecordComparator = TestData.getIntStringTupleComparator();
}
 
Example 24
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
	
	this.serializerFactory1 = TestData.getIntStringTupleSerializerFactory();
	this.comparator1 = TestData.getIntStringTupleComparator();

	this.serializerFactory2 = TestData.getIntIntTupleSerializerFactory();
	this.comparator2 = TestData.getIntIntTupleComparator();
}
 
Example 25
Source Project: Flink-CEPplus   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, MEMORY_PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
	this.serializer = new IntPairSerializer();
	this.comparator = new IntPairComparator();
}
 
Example 26
Source Project: Flink-CEPplus   Source File: HashVsSortMiniBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.serializer1 = TestData.getIntStringTupleSerializerFactory();
	this.serializer2 = TestData.getIntStringTupleSerializerFactory();
	this.comparator1 = TestData.getIntStringTupleComparator();
	this.comparator2 = TestData.getIntStringTupleComparator();
	this.pairComparator11 = new GenericPairComparator(this.comparator1, this.comparator2);
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 
Example 27
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteInputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: StreamMockEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
public StreamMockEnvironment(
	JobID jobID,
	ExecutionAttemptID executionAttemptID,
	Configuration jobConfig,
	Configuration taskConfig,
	ExecutionConfig executionConfig,
	long memorySize,
	MockInputSplitProvider inputSplitProvider,
	int bufferSize,
	TaskStateManager taskStateManager) {

	this.jobID = jobID;
	this.executionAttemptID = executionAttemptID;

	int subtaskIndex = 0;
	this.taskInfo = new TaskInfo(
		"", /* task name */
		1, /* num key groups / max parallelism */
		subtaskIndex, /* index of this subtask */
		1, /* num subtasks */
		0 /* attempt number */);
	this.jobConfiguration = jobConfig;
	this.taskConfiguration = taskConfig;
	this.inputs = new LinkedList<InputGate>();
	this.outputs = new LinkedList<ResultPartitionWriter>();
	this.memManager = new MemoryManager(memorySize, 1);
	this.ioManager = new IOManagerAsync();
	this.taskStateManager = Preconditions.checkNotNull(taskStateManager);
	this.aggregateManager = new TestGlobalAggregateManager();
	this.inputSplitProvider = inputSplitProvider;
	this.bufferSize = bufferSize;

	this.executionConfig = executionConfig;
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, getExecutionId());

	KvStateRegistry registry = new KvStateRegistry();
	this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
}
 
Example 30
/**
 * Sets up the environment including buffer pools and netty threads.
 *
 * @param writers
 * 		number of writers
 * @param channels
 * 		outgoing channels per writer
 * @param localMode
 * 		only local channels?
 * @param senderBufferPoolSize
 * 		buffer pool size for the sender (set to <tt>-1</tt> for default)
 * @param receiverBufferPoolSize
 * 		buffer pool size for the receiver (set to <tt>-1</tt> for default)
 */
public void setUp(
		int writers,
		int channels,
		boolean broadcastMode,
		boolean localMode,
		int senderBufferPoolSize,
		int receiverBufferPoolSize,
		Configuration config) throws Exception {
	this.broadcastMode = broadcastMode;
	this.localMode = localMode;
	this.channels = channels;
	this.partitionIds = new ResultPartitionID[writers];
	if (senderBufferPoolSize == -1) {
		senderBufferPoolSize = Math.max(2048, writers * channels * 4);
	}
	if (receiverBufferPoolSize == -1) {
		receiverBufferPoolSize = Math.max(2048, writers * channels * 4);
	}

	ioManager = new IOManagerAsync();

	senderEnv = createNettyNetworkEnvironment(senderBufferPoolSize, config);
	senderEnv.start();
	if (localMode && senderBufferPoolSize == receiverBufferPoolSize) {
		receiverEnv = senderEnv;
	}
	else {
		receiverEnv = createNettyNetworkEnvironment(receiverBufferPoolSize, config);
		receiverEnv.start();
	}

	generatePartitionIds();
}