org.apache.tez.dag.api.EdgeProperty.SchedulingType Java Examples

The following examples show how to use org.apache.tez.dag.api.EdgeProperty.SchedulingType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SimpleTestDAG.java From tez with Apache License 2.0

6 votes

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}

Example #2

Source File: TestEdge.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testInvalidSourceTaskIndex() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,1,1,-1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1),
        new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1",
            TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
    edge.sendTezEventToSourceTasks(ireEvent);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("SourceTaskIndex should not be negative"));
  }
}

Example #3

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.BROADCAST,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #4

Source File: SimpleTestDAG.java From incubator-tez with Apache License 2.0

6 votes

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}

Example #5

Source File: TestEdge.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testInvalidPhysicalOutputCount() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,-1,1,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    edge.getSourceSpec(0);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("PhysicalOutputCount should not be negative"));
  }
}

Example #6

Source File: TestEdge.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testInvalidPhysicalInputCount() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(-1,1,1,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    edge.getDestinationSpec(0);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("PhysicalInputCount should not be negative"));
  }
}

Example #7

Source File: TestMockDAGAppMaster.java From tez with Apache License 2.0

6 votes

private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
  DAG dag = DAG.create(dagName);
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
  Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
  uv12.addDataSink("uv12Out", uv12DataSink);
  DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
  v3.addDataSink("v3Out", v3DataSink);

  GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create("dummy output class"),
      InputDescriptor.create("dummy input class")), InputDescriptor
      .create("merge.class"));
  dag.addVertex(v1)
    .addVertex(v2)
    .addVertex(v3)
    .addEdge(e1);
  return dag;
}

Example #8

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      -1, dummyTaskResource);
  DataSourceDescriptor dsDesc = DataSourceDescriptor.create(InputDescriptor.create(dummyInputClassName),
      InputInitializerDescriptor.create(dummyInputInitClassName), null);
  v1.addDataSource("input_1", dsDesc);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}

Example #9

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerify4() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #10

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);

  v1.addDataSink("v2", DataSinkDescriptor.create(null, null, null));

  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #11

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #12

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #13

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyCustomEdge() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(new EdgeManagerDescriptor("emClass"),
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #14

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #15

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}

Example #16

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #17

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify3() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #18

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify4() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #19

Source File: TestAMRecovery.java From tez with Apache License 2.0

6 votes

/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}

Example #20

Source File: TestDAGVerify.java From incubator-tez with Apache License 2.0

6 votes

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #21

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);

  v2.addDataSource("v1", DataSourceDescriptor.create(null, null, null));

  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #22

Source File: TestDAGVerify.java From tez with Apache License 2.0

6 votes

@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}

Example #23

Source File: TestDAGPlan.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void testEdgeManagerSerde() {
  DAG dag = DAG.create("testDag");
  ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
  ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
  Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = InputDescriptor.create("input")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
  OutputDescriptor outputDescriptor = OutputDescriptor.create("output")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
  Edge edge = Edge.create(v1, v2, EdgeProperty.create(
      EdgeManagerPluginDescriptor.create("emClass").setUserPayload(
          UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))),
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);

  EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
      .getEdgeList().get(0));

  EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
  Assert.assertNotNull(emDesc);
  Assert.assertEquals("emClass", emDesc.getClassName());
  Assert.assertTrue(
      Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
}

Example #24

Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0

5 votes

@Test (timeout=60000)
public void testBasicSuccessBroadcast() throws Exception {
  DAG dag = new DAG("testBasicSuccessBroadcast");
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(null), 
          TestInput.getInputDesc(null))));
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}

Example #25

Source File: TwoLevelsFailingDAG.java From tez with Apache License 2.0

5 votes

/**
 * Adds an edge to given 2 vertices.
 * @param v1 vertice 1
 * @param v2 vertice 2
 * @param dataMovementType Data movement type
 */
protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) {
    dag.addEdge(Edge.create(v1, v2,
        EdgeProperty.create(dataMovementType,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
}

Example #26

Source File: TestPreemption.java From tez with Apache License 2.0

5 votes

DAG createDAG(DataMovementType dmType) {
  DAG dag = DAG.create("test-" + dagCount++);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
  Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 5);
  Edge eAB = Edge.create(vA, vB, 
  EdgeProperty.create(dmType, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, OutputDescriptor.create("O.class"),
      InputDescriptor.create("I.class")));
  
  dag.addVertex(vA).addVertex(vB).addEdge(eAB);
  return dag;
}

Example #27

Source File: TestDAGVerify.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
public void testVerifyOneToOneInferParallelism() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = Vertex.create("v3",
      ProcessorDescriptor.create("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  Edge e2 = Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.ONE_TO_ONE,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create(dummyOutputClassName),
          InputDescriptor.create(dummyInputClassName)));
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}

Example #28

Source File: TestDAG.java From tez with Apache License 2.0

5 votes

@Test(timeout = 5000)
public void testDuplicatedEdges() {
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Processor"),
      dummyTaskCount, dummyTaskResource);

  Edge edge1 = Edge.create(v1, v2, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.CONCURRENT, OutputDescriptor.create("output"),
      InputDescriptor.create("input")));
  Edge edge2 = Edge.create(v1, v2, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.CONCURRENT, OutputDescriptor.create("output"),
      InputDescriptor.create("input")));

  DAG dag = DAG.create("testDAG");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(edge1);
  try {
    dag.addEdge(edge2);
    Assert.fail("should fail it due to duplicated edges");
  } catch (Exception e) {
    e.printStackTrace();
    Assert.assertTrue(e.getMessage().contains("already defined!"));
  }
}

Example #29

Source File: SimpleVTestDAG.java From tez with Apache License 2.0

5 votes

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_V_DAG_NUM_TASKS, TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(Edge.create(v1, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}

Example #30

Source File: SimpleReverseVTestDAG.java From tez with Apache License 2.0

5 votes

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(Edge.create(v1, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}