Java Code Examples for org.apache.flink.runtime.operators.testutils.types.IntPairSerializer

The following examples show how to use org.apache.flink.runtime.operators.testutils.types.IntPairSerializer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example 3
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 4
Source Project: flink   Source File: ReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 5
Source Project: flink   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 
Example 6
Source Project: flink   Source File: NonReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 7
Source Project: flink   Source File: ReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 8
Source Project: flink   Source File: HashTableITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();

	this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 9
Source Project: flink   Source File: NonReusingHashJoinIteratorITCase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.ioManager = new IOManagerAsync();
}
 
Example 10
Source Project: Flink-CEPplus   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, MEMORY_PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
	this.serializer = new IntPairSerializer();
	this.comparator = new IntPairComparator();
}
 
Example 11
Source Project: flink   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, MEMORY_PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
	this.serializer = new IntPairSerializer();
	this.comparator = new IntPairComparator();
}
 
Example 12
Source Project: flink   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void beforeTest() {
	this.memoryManager = MemoryManagerBuilder
		.newBuilder()
		.setMemorySize(MEMORY_SIZE)
		.setPageSize(MEMORY_PAGE_SIZE)
		.build();
	this.ioManager = new IOManagerAsync();
	this.serializer = new IntPairSerializer();
	this.comparator = new IntPairComparator();
}
 
Example 13
Source Project: Flink-CEPplus   Source File: ExternalSortITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 14
Source Project: flink   Source File: ExternalSortITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 15
Source Project: flink   Source File: ExternalSortITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}