Java Code Examples for org.apache.flink.runtime.operators.testutils.DummyInvokable

The following examples show how to use org.apache.flink.runtime.operators.testutils.DummyInvokable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void allocateTooMuch() {
	try {
		final AbstractInvokable mockInvoke = new DummyInvokable();

		List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);

		try {
			this.memoryManager.allocatePages(mockInvoke, 1);
			Assert.fail("Expected MemoryAllocationException.");
		} catch (MemoryAllocationException maex) {
			// expected
		}

		Assert.assertTrue("The previously allocated segments were not valid any more.",
																allMemorySegmentsValid(segs));

		this.memoryManager.releaseAll(mockInvoke);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 2
Source Project: flink   Source File: MemoryManagerLazyAllocationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void allocateTooMuch() {
	try {
		final AbstractInvokable mockInvoke = new DummyInvokable();

		List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);

		try {
			this.memoryManager.allocatePages(mockInvoke, 1);
			Assert.fail("Expected MemoryAllocationException.");
		} catch (MemoryAllocationException maex) {
			// expected
		}

		Assert.assertTrue("The previously allocated segments were not valid any more.",
																allMemorySegmentsValid(segs));

		this.memoryManager.releaseAll(mockInvoke);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 3
Source Project: flink   Source File: MemoryManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void allocateTooMuch() {
	try {
		final AbstractInvokable mockInvoke = new DummyInvokable();

		List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);

		try {
			this.memoryManager.allocatePages(mockInvoke, 1);
			Assert.fail("Expected MemoryAllocationException.");
		} catch (MemoryAllocationException maex) {
			// expected
		}

		Assert.assertTrue("The previously allocated segments were not valid any more.",
																allMemorySegmentsValid(segs));

		this.memoryManager.releaseAll(mockInvoke);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 4
Source Project: flink   Source File: MemoryManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void allocateTooMuch() {
	try {
		final AbstractInvokable mockInvoke = new DummyInvokable();

		List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);

		testCannotAllocateAnymore(mockInvoke, 1);

		Assert.assertTrue("The previously allocated segments were not valid any more.",
																allMemorySegmentsValid(segs));

		this.memoryManager.releaseAll(mockInvoke);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 5
@Test
public void allocateMultipleOwners() {
	final int numOwners = 17;

	try {
		AbstractInvokable[] owners = new AbstractInvokable[numOwners];

		@SuppressWarnings("unchecked")
		List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[numOwners];

		for (int i = 0; i < numOwners; i++) {
			owners[i] = new DummyInvokable();
			mems[i] = new ArrayList<MemorySegment>(64);
		}

		// allocate all memory to the different owners
		for (int i = 0; i < NUM_PAGES; i++) {
			final int owner = this.random.nextInt(numOwners);
			mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
		}

		// free one owner at a time
		for (int i = 0; i < numOwners; i++) {
			this.memoryManager.releaseAll(owners[i]);
			owners[i] = null;
			Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
			mems[i] = null;

			// check that the owner owners were not affected
			for (int k = i + 1; k < numOwners; k++) {
				Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 6
Source Project: Flink-CEPplus   Source File: MemoryManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void allocateMultipleOwners() {
	final int numOwners = 17;

	try {
		AbstractInvokable[] owners = new AbstractInvokable[numOwners];

		@SuppressWarnings("unchecked")
		List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[numOwners];

		for (int i = 0; i < numOwners; i++) {
			owners[i] = new DummyInvokable();
			mems[i] = new ArrayList<MemorySegment>(64);
		}

		// allocate all memory to the different owners
		for (int i = 0; i < NUM_PAGES; i++) {
			final int owner = this.random.nextInt(numOwners);
			mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
		}

		// free one owner at a time
		for (int i = 0; i < numOwners; i++) {
			this.memoryManager.releaseAll(owners[i]);
			owners[i] = null;
			Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
			mems[i] = null;

			// check that the owner owners were not affected
			for (int k = i + 1; k < numOwners; k++) {
				Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: MemorySegmentSimpleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception{
	try {
		this.manager = new MemoryManager(MANAGED_MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
		this.segment = manager.allocatePages(new DummyInvokable(), 1).get(0);
		this.random = new Random(RANDOM_SEED);
	} catch (Exception e) {
		e.printStackTrace();
		fail("Test setup failed.");
	}
}
 
Example 8
Source Project: Flink-CEPplus   Source File: NormalizedKeySorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	do {
		generator.next(record);
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
	Tuple2<Integer, String> readTarget = new Tuple2<>();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		generator.next(record);
		
		int rk = readTarget.f0;
		int gk = record.f0;
		
		String rv = readTarget.f1;
		String gv = record.f1;
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: NormalizedKeySorterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, true);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 11
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteInputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 13
Source Project: flink   Source File: MemoryManagerLazyAllocationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void allocateMultipleOwners() {
	final int numOwners = 17;

	try {
		AbstractInvokable[] owners = new AbstractInvokable[numOwners];

		@SuppressWarnings("unchecked")
		List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[numOwners];

		for (int i = 0; i < numOwners; i++) {
			owners[i] = new DummyInvokable();
			mems[i] = new ArrayList<MemorySegment>(64);
		}

		// allocate all memory to the different owners
		for (int i = 0; i < NUM_PAGES; i++) {
			final int owner = this.random.nextInt(numOwners);
			mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
		}

		// free one owner at a time
		for (int i = 0; i < numOwners; i++) {
			this.memoryManager.releaseAll(owners[i]);
			owners[i] = null;
			Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
			mems[i] = null;

			// check that the owner owners were not affected
			for (int k = i + 1; k < numOwners; k++) {
				Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 14
Source Project: flink   Source File: MemoryManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void allocateMultipleOwners() {
	final int numOwners = 17;

	try {
		AbstractInvokable[] owners = new AbstractInvokable[numOwners];

		@SuppressWarnings("unchecked")
		List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[numOwners];

		for (int i = 0; i < numOwners; i++) {
			owners[i] = new DummyInvokable();
			mems[i] = new ArrayList<MemorySegment>(64);
		}

		// allocate all memory to the different owners
		for (int i = 0; i < NUM_PAGES; i++) {
			final int owner = this.random.nextInt(numOwners);
			mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
		}

		// free one owner at a time
		for (int i = 0; i < numOwners; i++) {
			this.memoryManager.releaseAll(owners[i]);
			owners[i] = null;
			Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
			mems[i] = null;

			// check that the owner owners were not affected
			for (int k = i + 1; k < numOwners; k++) {
				Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 15
Source Project: flink   Source File: MemorySegmentSimpleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception{
	try {
		this.manager = new MemoryManager(MANAGED_MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
		this.segment = manager.allocatePages(new DummyInvokable(), 1).get(0);
		this.random = new Random(RANDOM_SEED);
	} catch (Exception e) {
		e.printStackTrace();
		fail("Test setup failed.");
	}
}
 
Example 16
Source Project: flink   Source File: NormalizedKeySorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	do {
		generator.next(record);
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
	Tuple2<Integer, String> readTarget = new Tuple2<>();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		generator.next(record);
		
		int rk = readTarget.f0;
		int gk = record.f0;
		
		String rv = readTarget.f1;
		String gv = record.f1;
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 17
Source Project: flink   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteInputView() {
	try (IOManager ioManager = new IOManagerAsync()) {
		MemoryManager memMan = MemoryManagerBuilder.newBuilder().build();
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 18
Source Project: flink   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, true);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 19
Source Project: flink   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	try (IOManager ioManager = new IOManagerAsync()) {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 20
Source Project: flink   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteInputView() {
	try (IOManager ioManager = new IOManagerAsync()) {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 21
Source Project: AthenaX   Source File: JobITestUtil.java    License: Apache License 2.0 5 votes vote down vote up
static JobGraph trivialJobGraph() {
  JobGraph g = new JobGraph();
  JobVertex v = new JobVertex("1");
  v.setInvokableClass(DummyInvokable.class);
  g.addVertex(v);
  return g;
}
 
Example 22
Source Project: flink   Source File: MemoryManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void allocateMultipleOwners() {
	final int numOwners = 17;

	try {
		AbstractInvokable[] owners = new AbstractInvokable[numOwners];

		@SuppressWarnings("unchecked")
		List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[numOwners];

		for (int i = 0; i < numOwners; i++) {
			owners[i] = new DummyInvokable();
			mems[i] = new ArrayList<MemorySegment>(64);
		}

		// allocate all memory to the different owners
		for (int i = 0; i < NUM_PAGES; i++) {
			final int owner = this.random.nextInt(numOwners);
			mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
		}

		// free one owner at a time
		for (int i = 0; i < numOwners; i++) {
			this.memoryManager.releaseAll(owners[i]);
			owners[i] = null;
			Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
			mems[i] = null;

			// check that the owner owners were not affected
			for (int k = i + 1; k < numOwners; k++) {
				Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 23
Source Project: flink   Source File: MemoryManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void doubleReleaseReturnsMemoryOnlyOnce() throws MemoryAllocationException {
	final AbstractInvokable mockInvoke = new DummyInvokable();

	Collection<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);
	MemorySegment segment = segs.iterator().next();

	this.memoryManager.release(segment);
	this.memoryManager.release(segment);

	testCannotAllocateAnymore(mockInvoke, 2);

	this.memoryManager.releaseAll(mockInvoke);
}
 
Example 24
Source Project: flink   Source File: MemorySegmentSimpleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception{
	try {
		this.manager = MemoryManagerBuilder
			.newBuilder()
			.setMemorySize(MANAGED_MEMORY_SIZE)
			.setPageSize(PAGE_SIZE)
			.build();
		this.segment = manager.allocatePages(new DummyInvokable(), 1).get(0);
		this.random = new Random(RANDOM_SEED);
	} catch (Exception e) {
		e.printStackTrace();
		fail("Test setup failed.");
	}
}
 
Example 25
Source Project: flink   Source File: NormalizedKeySorterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	do {
		generator.next(record);
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
	Tuple2<Integer, String> readTarget = new Tuple2<>();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		generator.next(record);
		
		int rk = readTarget.f0;
		int gk = record.f0;
		
		String rv = readTarget.f1;
		String gv = record.f1;
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 26
Source Project: flink   Source File: NormalizedKeySorterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 27
Source Project: flink   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, true);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
Example 28
Source Project: flink   Source File: FileChannelStreamsITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testWriteReadSmallRecords() {
	try {
		List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
		
		final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		final FileIOChannel.ID channel = ioManager.createChannel();
		
		// create the writer output view
		final BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
		
		// write a number of pairs
		Pair pair = new Pair();
		for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
			generator.next(pair);
			pair.write(outView);
		}
		outView.close();
		
		// create the reader input view
		List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
		
		final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
		generator.reset();
		
		// read and re-generate all records and compare them
		Pair readPair = new Pair();
		for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
			generator.next(pair);
			readPair.read(inView);
			assertEquals("The re-generated and the read record do not match.", pair, readPair);
		}
		
		inView.close();
		reader.deleteChannel();
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: JsonGeneratorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testGeneratorWithoutAnyAttachements() {
	try {
		JobVertex source1 = new JobVertex("source 1");
		
		JobVertex source2 = new JobVertex("source 2");
		source2.setInvokableClass(DummyInvokable.class);
		
		JobVertex source3 = new JobVertex("source 3");
		
		JobVertex intermediate1 = new JobVertex("intermediate 1");
		JobVertex intermediate2 = new JobVertex("intermediate 2");
		
		JobVertex join1 = new JobVertex("join 1");
		JobVertex join2 = new JobVertex("join 2");

		JobVertex sink1 = new JobVertex("sink 1");
		JobVertex sink2 = new JobVertex("sink 2");
		
		intermediate1.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
		intermediate2.connectNewDataSetAsInput(source2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
		
		join1.connectNewDataSetAsInput(intermediate1, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
		join1.connectNewDataSetAsInput(intermediate2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

		join2.connectNewDataSetAsInput(join1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
		join2.connectNewDataSetAsInput(source3, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
		
		sink1.connectNewDataSetAsInput(join2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
		sink2.connectNewDataSetAsInput(join1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

		JobGraph jg = new JobGraph("my job", source1, source2, source3,
				intermediate1, intermediate2, join1, join2, sink1, sink2);
		
		String plan = JsonPlanGenerator.generatePlan(jg);
		assertNotNull(plan);

		// validate the produced JSON
		ObjectMapper m = new ObjectMapper();
		JsonNode rootNode = m.readTree(plan);
		
		// core fields
		assertEquals(new TextNode(jg.getJobID().toString()), rootNode.get("jid"));
		assertEquals(new TextNode(jg.getName()), rootNode.get("name"));
		
		assertTrue(rootNode.path("nodes").isArray());
		
		for (Iterator<JsonNode> iter = rootNode.path("nodes").elements(); iter.hasNext(); ){
			JsonNode next = iter.next();
			
			JsonNode idNode = next.get("id");
			assertNotNull(idNode);
			assertTrue(idNode.isTextual());
			checkVertexExists(idNode.asText(), jg);
			
			String description = next.get("description").asText();
			assertTrue(
					description.startsWith("source") ||
					description.startsWith("sink") ||
					description.startsWith("intermediate") ||
					description.startsWith("join"));
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 30
Source Project: flink   Source File: FixedLengthRecordSorterTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testFlushPartialMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 1, NUM_RECORDS - 1);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =1;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}