org.apache.flink.core.io.LocatableInputSplit Java Examples

The following examples show how to use org.apache.flink.core.io.LocatableInputSplit. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractLocatableParallelSource.java    From alibaba-flink-connectors with Apache License 2.0 5 votes vote down vote up
@Override
public InputSplitAssigner getInputSplitAssigner(InputSplit[] inputSplits) {
	LocatableInputSplit[] locatableInputSplits = new LocatableInputSplit[inputSplits.length];
	for (int i = 0; i < inputSplits.length; i++) {
		locatableInputSplits[i] = (LocatableInputSplit) inputSplits[i];
	}
	if (source.isParallelReadDisabled()) {
		return new LocatableInputSplitAssigner(locatableInputSplits);
	}
	return super.getInputSplitAssigner(inputSplits);
}
 
Example #2
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForRemoteHost() {
	try {
		final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
		final int NUM_SPLITS = 10 * hosts.length;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("anotherHost", 0));
		
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #3
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForSameHost() {
	try {
		final int NUM_SPLITS = 50;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, "testhost"));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		
		assertEquals(0, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #4
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentWithNullHost() {
	try {
		final int NUM_SPLITS = 50;
		final String[][] hosts = new String[][] {
				new String[] { "localhost" },
				new String[0],
				null
		};
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%3]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit(null, 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #5
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void returnInputSplit(List<InputSplit> splits, int taskId) {
	synchronized (this.unassigned) {
		for (InputSplit split : splits) {
			LocatableInputSplitWithCount lisw = new LocatableInputSplitWithCount((LocatableInputSplit) split);
			this.remoteSplitChooser.addInputSplit(lisw);
			this.unassigned.add(lisw);
		}
	}
}
 
Example #6
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForRemoteHost() {
	try {
		final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
		final int NUM_SPLITS = 10 * hosts.length;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("anotherHost", 0));
		
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #7
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForSameHost() {
	try {
		final int NUM_SPLITS = 50;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, "testhost"));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		
		assertEquals(0, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #8
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentWithNullHost() {
	try {
		final int NUM_SPLITS = 50;
		final String[][] hosts = new String[][] {
				new String[] { "localhost" },
				new String[0],
				null
		};
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%3]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit(null, 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #9
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void returnInputSplit(List<InputSplit> splits, int taskId) {
	synchronized (this.unassigned) {
		for (InputSplit split : splits) {
			LocatableInputSplitWithCount lisw = new LocatableInputSplitWithCount((LocatableInputSplit) split);
			this.remoteSplitChooser.addInputSplit(lisw);
			this.unassigned.add(lisw);
		}
	}
}
 
Example #10
Source File: LocatableSplitAssignerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentWithNullHost() {
	try {
		final int NUM_SPLITS = 50;
		final String[][] hosts = new String[][] {
				new String[] { "localhost" },
				new String[0],
				null
		};
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%3]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit(null, 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #11
Source File: LocatableSplitAssignerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForRemoteHost() {
	try {
		final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
		final int NUM_SPLITS = 10 * hosts.length;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("anotherHost", 0));
		
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #12
Source File: LocatableSplitAssignerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialSplitAssignmentAllForSameHost() {
	try {
		final int NUM_SPLITS = 50;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, "testhost"));
		}
		
		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		InputSplit is = null;
		while ((is = ia.getNextInputSplit("testhost", 0)) != null) {
			assertTrue(splits.remove(is));
		}
		
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("", 0));
		
		assertEquals(0, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #13
Source File: LocatableInputSplitAssigner.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitAssigner(Collection<LocatableInputSplit> splits) {
	for(LocatableInputSplit split : splits) {
		this.unassigned.add(new LocatableInputSplitWithCount(split));
	}
	this.remoteSplitChooser = new LocatableInputSplitChooser(unassigned);
}
 
Example #14
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitAssigner(Collection<LocatableInputSplit> splits) {
	for(LocatableInputSplit split : splits) {
		this.unassigned.add(new LocatableInputSplitWithCount(split));
	}
	this.remoteSplitChooser = new LocatableInputSplitChooser(unassigned);
}
 
Example #15
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitAssigner(LocatableInputSplit[] splits) {
	for(LocatableInputSplit split : splits) {
		this.unassigned.add(new LocatableInputSplitWithCount(split));
	}
	this.remoteSplitChooser = new LocatableInputSplitChooser(unassigned);
}
 
Example #16
Source File: LocatableInputSplitAssigner.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitWithCount(LocatableInputSplit split) {
	this.split = split;
	this.localCount = 0;
}
 
Example #17
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitWithCount(LocatableInputSplit split) {
	this.split = split;
	this.localCount = 0;
}
 
Example #18
Source File: LocatableInputSplitAssigner.java    From flink with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplit getSplit() {
	return this.split;
}
 
Example #19
Source File: LocatableInputSplitAssigner.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public LocatableInputSplitAssigner(LocatableInputSplit[] splits) {
	for(LocatableInputSplit split : splits) {
		this.unassigned.add(new LocatableInputSplitWithCount(split));
	}
	this.remoteSplitChooser = new LocatableInputSplitChooser(unassigned);
}
 
Example #20
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentForSingleHost() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, "testhost"));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit("testhost", 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("testhost", 0));
		
		assertEquals(0, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #21
Source File: AbstractLocatableParallelSource.java    From alibaba-flink-connectors with Apache License 2.0 4 votes vote down vote up
@Override
public abstract LocatableInputSplit[] createInputSplitsForCurrentSubTask(int numberOfParallelSubTasks, int indexOfThisSubTask)
		throws IOException;
 
Example #22
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSerialSplitAssignmentMultiLocalHost() {
	try {

		final String[] localHosts = { "local1", "local2", "local3" };
		final String[] remoteHosts = { "remote1", "remote2", "remote3" };
		final String[] requestingHosts = { "local3", "local2", "local1", "other" };

		final int NUM_THREE_LOCAL_SPLITS = 10;
		final int NUM_TWO_LOCAL_SPLITS = 10;
		final int NUM_ONE_LOCAL_SPLITS = 10;
		final int NUM_LOCAL_SPLITS = 30;
		final int NUM_REMOTE_SPLITS = 10;
		final int NUM_SPLITS = 40;

		String[] threeLocalHosts = localHosts;
		String[] twoLocalHosts = {localHosts[0], localHosts[1], remoteHosts[0]};
		String[] oneLocalHost = {localHosts[0], remoteHosts[0], remoteHosts[1]};
		String[] noLocalHost = remoteHosts;

		int splitCnt = 0;
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		// add splits with three local hosts
		for (int i = 0; i < NUM_THREE_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, threeLocalHosts));
		}
		// add splits with two local hosts
					for (int i = 0; i < NUM_TWO_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, twoLocalHosts));
		}
		// add splits with two local hosts
		for (int i = 0; i < NUM_ONE_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, oneLocalHost));
		}
		// add splits with two local hosts
		for (int i = 0; i < NUM_REMOTE_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, noLocalHost));
		}

		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		LocatableInputSplit is = null;
		for (int i = 0; i < NUM_SPLITS; i++) {
			String host = requestingHosts[i % requestingHosts.length];
			is = ia.getNextInputSplit(host, 0);
			// check valid split
			assertTrue(is != null);
			// check unassigned split
			assertTrue(splits.remove(is));
			// check priority of split
			if (host.equals(localHosts[0])) {
				assertTrue(Arrays.equals(is.getHostnames(), oneLocalHost));
			} else if (host.equals(localHosts[1])) {
				assertTrue(Arrays.equals(is.getHostnames(), twoLocalHosts));
			} else if (host.equals(localHosts[2])) {
				assertTrue(Arrays.equals(is.getHostnames(), threeLocalHosts));
			} else {
				assertTrue(Arrays.equals(is.getHostnames(), noLocalHost));
			}
		}
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("anotherHost", 0));

		assertEquals(NUM_REMOTE_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_LOCAL_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #23
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentNullHost() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		final String[][] hosts = new String[][] {
				new String[] { "localhost" },
				new String[0],
				null
		};
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%3]));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit(null, 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("", 0));
		
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #24
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentForSingleHost() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, "testhost"));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit("testhost", 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("testhost", 0));
		
		assertEquals(0, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #25
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentForMultipleHosts() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%hosts.length]));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				final String threadHost = hosts[(int) (Math.random() * hosts.length)];
				
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit(threadHost, 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("testhost", 0));
		
		// at least one fraction of hosts needs be local, no matter how bad the thread races
		assertTrue(ia.getNumberOfLocalAssignments() >= NUM_SPLITS / hosts.length);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #26
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testAssignmentOfManySplitsRandomly() {

	long seed = Calendar.getInstance().getTimeInMillis();

	final int NUM_SPLITS = 65536;
	final String[] splitHosts = new String[256];
	final String[] requestingHosts = new String[256];
	final Random rand = new Random(seed);

	for (int i = 0; i < splitHosts.length; i++) {
		splitHosts[i] = "localHost" + i;
	}
	for (int i = 0; i < requestingHosts.length; i++) {
		if (i % 2 == 0) {
			requestingHosts[i] = "localHost" + i;
		} else {
			requestingHosts[i] = "remoteHost" + i;
		}
	}

	String[] stringArray = {};
	Set<String> hosts = new HashSet<String>();
	Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
	for (int i = 0; i < NUM_SPLITS; i++) {
		while (hosts.size() < 3) {
			hosts.add(splitHosts[rand.nextInt(splitHosts.length)]);
		}
		splits.add(new LocatableInputSplit(i, hosts.toArray(stringArray)));
		hosts.clear();
	}

	final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);

	for (int i = 0; i < NUM_SPLITS; i++) {
		LocatableInputSplit split = ia.getNextInputSplit(requestingHosts[rand.nextInt(requestingHosts.length)], 0);
		assertTrue(split != null);
		assertTrue(splits.remove(split));
	}

	assertTrue(splits.isEmpty());
	assertNull(ia.getNextInputSplit("testHost", 0));
}
 
Example #27
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testAssignmentOfManySplitsRandomly() {

	long seed = Calendar.getInstance().getTimeInMillis();

	final int NUM_SPLITS = 65536;
	final String[] splitHosts = new String[256];
	final String[] requestingHosts = new String[256];
	final Random rand = new Random(seed);

	for (int i = 0; i < splitHosts.length; i++) {
		splitHosts[i] = "localHost" + i;
	}
	for (int i = 0; i < requestingHosts.length; i++) {
		if (i % 2 == 0) {
			requestingHosts[i] = "localHost" + i;
		} else {
			requestingHosts[i] = "remoteHost" + i;
		}
	}

	String[] stringArray = {};
	Set<String> hosts = new HashSet<String>();
	Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
	for (int i = 0; i < NUM_SPLITS; i++) {
		while (hosts.size() < 3) {
			hosts.add(splitHosts[rand.nextInt(splitHosts.length)]);
		}
		splits.add(new LocatableInputSplit(i, hosts.toArray(stringArray)));
		hosts.clear();
	}

	final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);

	for (int i = 0; i < NUM_SPLITS; i++) {
		LocatableInputSplit split = ia.getNextInputSplit(requestingHosts[rand.nextInt(requestingHosts.length)], 0);
		assertTrue(split != null);
		assertTrue(splits.remove(split));
	}

	assertTrue(splits.isEmpty());
	assertNull(ia.getNextInputSplit("testHost", 0));
}
 
Example #28
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentForMultipleHosts() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" };
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%hosts.length]));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				final String threadHost = hosts[(int) (Math.random() * hosts.length)];
				
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit(threadHost, 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("testhost", 0));
		
		// at least one fraction of hosts needs be local, no matter how bad the thread races
		assertTrue(ia.getNumberOfLocalAssignments() >= NUM_SPLITS / hosts.length);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #29
Source File: LocatableSplitAssignerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSerialSplitAssignmentMultiLocalHost() {
	try {

		final String[] localHosts = { "local1", "local2", "local3" };
		final String[] remoteHosts = { "remote1", "remote2", "remote3" };
		final String[] requestingHosts = { "local3", "local2", "local1", "other" };

		final int NUM_THREE_LOCAL_SPLITS = 10;
		final int NUM_TWO_LOCAL_SPLITS = 10;
		final int NUM_ONE_LOCAL_SPLITS = 10;
		final int NUM_LOCAL_SPLITS = 30;
		final int NUM_REMOTE_SPLITS = 10;
		final int NUM_SPLITS = 40;

		String[] threeLocalHosts = localHosts;
		String[] twoLocalHosts = {localHosts[0], localHosts[1], remoteHosts[0]};
		String[] oneLocalHost = {localHosts[0], remoteHosts[0], remoteHosts[1]};
		String[] noLocalHost = remoteHosts;

		int splitCnt = 0;
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		// add splits with three local hosts
		for (int i = 0; i < NUM_THREE_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, threeLocalHosts));
		}
		// add splits with two local hosts
					for (int i = 0; i < NUM_TWO_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, twoLocalHosts));
		}
		// add splits with two local hosts
		for (int i = 0; i < NUM_ONE_LOCAL_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, oneLocalHost));
		}
		// add splits with two local hosts
		for (int i = 0; i < NUM_REMOTE_SPLITS; i++) {
			splits.add(new LocatableInputSplit(splitCnt++, noLocalHost));
		}

		// get all available splits
		LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		LocatableInputSplit is = null;
		for (int i = 0; i < NUM_SPLITS; i++) {
			String host = requestingHosts[i % requestingHosts.length];
			is = ia.getNextInputSplit(host, 0);
			// check valid split
			assertTrue(is != null);
			// check unassigned split
			assertTrue(splits.remove(is));
			// check priority of split
			if (host.equals(localHosts[0])) {
				assertTrue(Arrays.equals(is.getHostnames(), oneLocalHost));
			} else if (host.equals(localHosts[1])) {
				assertTrue(Arrays.equals(is.getHostnames(), twoLocalHosts));
			} else if (host.equals(localHosts[2])) {
				assertTrue(Arrays.equals(is.getHostnames(), threeLocalHosts));
			} else {
				assertTrue(Arrays.equals(is.getHostnames(), noLocalHost));
			}
		}
		// check we had all
		assertTrue(splits.isEmpty());
		assertNull(ia.getNextInputSplit("anotherHost", 0));

		assertEquals(NUM_REMOTE_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(NUM_LOCAL_SPLITS, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #30
Source File: LocatableSplitAssignerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConcurrentSplitAssignmentNullHost() {
	try {
		final int NUM_THREADS = 10;
		final int NUM_SPLITS = 500;
		final int SUM_OF_IDS = (NUM_SPLITS-1) * (NUM_SPLITS) / 2;
		
		final String[][] hosts = new String[][] {
				new String[] { "localhost" },
				new String[0],
				null
		};
		
		// load some splits
		Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>();
		for (int i = 0; i < NUM_SPLITS; i++) {
			splits.add(new LocatableInputSplit(i, hosts[i%3]));
		}
		
		final LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits);
		
		final AtomicInteger splitsRetrieved = new AtomicInteger(0);
		final AtomicInteger sumOfIds = new AtomicInteger(0);
		
		Runnable retriever = new Runnable() {
			
			@Override
			public void run() {
				LocatableInputSplit split;
				while ((split = ia.getNextInputSplit(null, 0)) != null) {
					splitsRetrieved.incrementAndGet();
					sumOfIds.addAndGet(split.getSplitNumber());
				}
			}
		};
		
		// create the threads
		Thread[] threads = new Thread[NUM_THREADS];
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i] = new Thread(retriever);
			threads[i].setDaemon(true);
		}
		
		// launch concurrently
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].start();
		}
		
		// sync
		for (int i = 0; i < NUM_THREADS; i++) {
			threads[i].join(5000);
		}
		
		// verify
		for (int i = 0; i < NUM_THREADS; i++) {
			if (threads[i].isAlive()) {
				fail("The concurrency test case is erroneous, the thread did not respond in time.");
			}
		}
		
		assertEquals(NUM_SPLITS, splitsRetrieved.get());
		assertEquals(SUM_OF_IDS, sumOfIds.get());
		
		// nothing left
		assertNull(ia.getNextInputSplit("", 0));
		
		assertEquals(NUM_SPLITS, ia.getNumberOfRemoteAssignments());
		assertEquals(0, ia.getNumberOfLocalAssignments());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}