org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator Java Examples

The following examples show how to use org.apache.flink.runtime.operators.testutils.RandomIntPairGenerator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #2
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current;
	int last;
	
	iter.next(readTarget);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		last = current;
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #3
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #4
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #5
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #6
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}
 
Example #7
Source File: ExternalSortITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #8
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current;
	int last;
	
	iter.next(readTarget);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		last = current;
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #9
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #10
Source File: ExternalSortITCase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #11
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #12
Source File: FixedLengthRecordSorterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}
 
Example #13
Source File: ExternalSortITCase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example #14
Source File: FixedLengthRecordSorterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current;
	int last;
	
	iter.next(readTarget);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		last = current;
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #15
Source File: FixedLengthRecordSorterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #16
Source File: FixedLengthRecordSorterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #17
Source File: FixedLengthRecordSorterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example #18
Source File: FixedLengthRecordSorterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}