Java Code Examples for org.apache.flink.runtime.JobException#printStackTrace()

The following examples show how to use org.apache.flink.runtime.JobException#printStackTrace() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test2NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(2 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(2, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 2, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 2 + 1, inEdges[1].getSource().getPartitionNumber());
	}
}

Example 2

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNToN() throws Exception {
	final int N = 23;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex(), inEdges[0].getSource().getPartitionNumber());
	}
}

Example 3

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNToN() throws Exception {
	final int N = 23;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex(), inEdges[0].getSource().getPartitionNumber());
	}
}

Example 4

Source File: PointwisePatternTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void test2NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(2 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(2, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 2, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 2 + 1, inEdges[1].getSource().getPartitionNumber());
	}
}

Example 5

Source File: PointwisePatternTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void test3NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(3 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(3, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
	}
}

Example 6

Source File: PointwisePatternTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testNTo2N() throws Exception {
	final int N = 41;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(2 * N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() / 2, inEdges[0].getSource().getPartitionNumber());
	}
}

Example 7

Source File: ExecutionGraphConstructionTest.java From flink with Apache License 2.0

5 votes

/**
 * Creates a JobGraph of the following form:
 * 
 * <pre>
 *  v1--->v2-->\
 *              \
 *               v4 --->\
 *        ----->/        \
 *  v3-->/                v5
 *       \               /
 *        ------------->/
 * </pre>
 */
@Test
public void testCreateSimpleGraphBipartite() throws Exception {
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");
	
	v1.setParallelism(5);
	v2.setParallelism(7);
	v3.setParallelism(2);
	v4.setParallelism(11);
	v5.setParallelism(4);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);
	v5.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	
	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

	ExecutionGraph eg = createExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	verifyTestGraph(eg, v1, v2, v3, v4, v5);
}

Example 8

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test3NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(3 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(3, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
	}
}

Example 9

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNTo2N() throws Exception {
	final int N = 41;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(N);
	v2.setParallelism(2 * N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() / 2, inEdges[0].getSource().getPartitionNumber());
	}
}

Example 10

Source File: PointwisePatternTest.java From flink with Apache License 2.0

5 votes

@Test
public void test3NToN() throws Exception {
	final int N = 17;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(3 * N);
	v2.setParallelism(N);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(3, inEdges.length);
		
		assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
		assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
	}
}

Example 11

Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0

4 votes

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v1-->v2-->|\
    *              \
    *               v4
    *             /
    *  v3------>/
    * </pre>
    */
@Test
public void testMultiRegionNotAllToAll() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(2);
       v2.setParallelism(2);
       v3.setParallelism(5);
       v4.setParallelism(5);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);

       v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
       v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4));

	final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

       ExecutionGraph eg = new ExecutionGraph(
       	jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]);
       FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);

       //There should be 3 failover regions. v1 v2 in two, v3 and v4 in one
       assertEquals(region11, region21);
       assertEquals(region12, region22);
       assertFalse(region11.equals(region12));
       assertFalse(region3.equals(region4));
}

Example 12

Source File: PointwisePatternTest.java From flink with Apache License 2.0

4 votes

private void testHighToLow(int highDop, int lowDop) throws Exception {
	if (highDop < lowDop) {
		throw new IllegalArgumentException();
	}
	
	final int factor = highDop / lowDop;
	final int delta = highDop % lowDop == 0 ? 0 : 1;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(highDop);
	v2.setParallelism(lowDop);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	int[] timesUsed = new int[highDop];
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertTrue(inEdges.length >= factor && inEdges.length <= factor + delta);
		
		for (ExecutionEdge ee : inEdges) {
			timesUsed[ee.getSource().getPartitionNumber()]++;
		}
	}

	for (int used : timesUsed) {
		assertEquals(1, used);
	}
}

Example 13

Source File: FailoverRegionTest.java From flink with Apache License 2.0

4 votes

private ExecutionGraph createSingleRegionExecutionGraph(RestartStrategy restartStrategy) throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 14);

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");

	v1.setParallelism(3);
	v2.setParallelism(2);
	v3.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v3.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<>(Arrays.asList(v1, v2, v3));

	ExecutionGraph eg = new ExecutionGraph(
		new DummyJobInformation(
			jobId,
			jobName),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		new FailoverPipelinedRegionWithDirectExecutor(),
		slotProvider);
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

	enableCheckpointing(eg);

	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	eg.scheduleForExecution();

	attachPendingCheckpoints(eg);
	return eg;
}

Example 14

Source File: PointwisePatternTest.java From flink with Apache License 2.0

4 votes

private void testLowToHigh(int lowDop, int highDop) throws Exception {
	if (highDop < lowDop) {
		throw new IllegalArgumentException();
	}
	
	final int factor = highDop / lowDop;
	final int delta = highDop % lowDop == 0 ? 0 : 1;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(lowDop);
	v2.setParallelism(highDop);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	int[] timesUsed = new int[lowDop];
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		
		timesUsed[inEdges[0].getSource().getPartitionNumber()]++;
	}

	for (int used : timesUsed) {
		assertTrue(used >= factor && used <= factor + delta);
	}
}

Example 15

Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v1--->v2-->\
    *              \
    *               v4 --->|\
    *        ----->/        \
    *  v3-->/                v5
    *       \               /
    *        ------------->/
    * </pre>
    */
@Test
public void testSingleRegionWithMixedInput() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(3);
       v2.setParallelism(2);
       v3.setParallelism(2);
       v4.setParallelism(5);
       v5.setParallelism(2);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);
       v5.setInvokableClass(AbstractInvokable.class);

       v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

	final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       assertEquals(region1, region2);
       assertEquals(region2, region4);
       assertEquals(region3, region2);
       assertEquals(region1, region5);
   }

Example 16

Source File: RestartPipelinedRegionStrategyTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v2 ------->\
    *              \
    *  v1---------> v4 --->|\
    *                        \
    *                        v5
    *                       /
    *  v3--------------->|/
    * </pre>
    */
@Test
public void testMultipleFailoverRegions() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(3);
       v2.setParallelism(2);
       v3.setParallelism(2);
       v4.setParallelism(5);
       v5.setParallelism(2);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);
       v5.setInvokableClass(AbstractInvokable.class);

       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
       v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

       final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
       FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]);
       FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]);
       FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       //There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two
       assertEquals(region1, region2);
       assertEquals(region2, region4);
       assertFalse(region31.equals(region32));
       assertFalse(region51.equals(region52));
}

Example 17

Source File: FailoverRegionTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests that two failover regions failover at the same time, they will not influence each other
 * @throws Exception
 */
@Test
public void testMultiRegionFailoverAtSameTime() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 16);

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");

	v1.setParallelism(2);
	v2.setParallelism(2);
	v3.setParallelism(2);
	v4.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);

	ExecutionGraph eg = new ExecutionGraph(
			new DummyJobInformation(
				jobId,
				jobName),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			AkkaUtils.getDefaultTimeout(),
			new InfiniteDelayRestartStrategy(10),
			new RestartPipelinedRegionStrategy.Factory(),
			slotProvider);
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	eg.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	eg.scheduleForExecution();
	RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();

	ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0];
	ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1];
	ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
	ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1];
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());

	ev11.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	ev31.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState());

	ev32.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev31), JobStatus.RUNNING, 1000);

	ev12.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev11), JobStatus.RUNNING, 1000);
}

Example 18

Source File: PointwisePatternTest.java From flink with Apache License 2.0

4 votes

private void testLowToHigh(int lowDop, int highDop) throws Exception {
	if (highDop < lowDop) {
		throw new IllegalArgumentException();
	}
	
	final int factor = highDop / lowDop;
	final int delta = highDop % lowDop == 0 ? 0 : 1;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(lowDop);
	v2.setParallelism(highDop);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	int[] timesUsed = new int[lowDop];
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertEquals(1, inEdges.length);
		
		
		timesUsed[inEdges[0].getSource().getPartitionNumber()]++;
	}

	for (int used : timesUsed) {
		assertTrue(used >= factor && used <= factor + delta);
	}
}

Example 19

Source File: PointwisePatternTest.java From flink with Apache License 2.0

4 votes

private void testHighToLow(int highDop, int lowDop) throws Exception {
	if (highDop < lowDop) {
		throw new IllegalArgumentException();
	}
	
	final int factor = highDop / lowDop;
	final int delta = highDop % lowDop == 0 ? 0 : 1;
	
	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");

	v1.setParallelism(highDop);
	v2.setParallelism(lowDop);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));

	ExecutionGraph eg = getDummyExecutionGraph();
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	
	ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
	
	int[] timesUsed = new int[highDop];
	
	for (ExecutionVertex ev : target.getTaskVertices()) {
		assertEquals(1, ev.getNumberOfInputs());
		
		ExecutionEdge[] inEdges = ev.getInputEdges(0);
		assertTrue(inEdges.length >= factor && inEdges.length <= factor + delta);
		
		for (ExecutionEdge ee : inEdges) {
			timesUsed[ee.getSource().getPartitionNumber()]++;
		}
	}

	for (int used : timesUsed) {
		assertEquals(1, used);
	}
}

Example 20

Source File: RestartPipelinedRegionStrategyTest.java From flink with Apache License 2.0

4 votes

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v1--->v2-->\
    *              \
    *               v4 --->|\
    *        ----->/        \
    *  v3-->/                v5
    *       \               /
    *        ------------->/
    * </pre>
    */
@Test
public void testSingleRegionWithMixedInput() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(3);
       v2.setParallelism(2);
       v3.setParallelism(2);
       v4.setParallelism(5);
       v5.setParallelism(2);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);
       v5.setInvokableClass(AbstractInvokable.class);

       v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

	final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       assertEquals(region1, region2);
       assertEquals(region2, region4);
       assertEquals(region3, region2);
       assertEquals(region1, region5);
   }