Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#validate()

@Test
public void validatePositiveOiO()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  RecoverableInputOperator inputOperator = plan.addOperator("IntegerGenerator", new RecoverableInputOperator());
  CollectorOperator outputOperator = plan.addOperator("IntegerCollector", new CollectorOperator());
  plan.addStream("PossibleOiO", inputOperator.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);

  try {
    plan.validate();
    Assert.assertTrue("OiO validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIO Single InputPort");
  }
}
 

@Test
public void testSQLSelectInsertWithAPI() throws IOException, ClassNotFoundException
{
  LogicalPlan dag = new LogicalPlan();

  String schemaIn = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" +
      "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss\"}}," +
      "{\"name\":\"id\",\"type\":\"Integer\"}," +
      "{\"name\":\"Product\",\"type\":\"String\"}," +
      "{\"name\":\"units\",\"type\":\"Integer\"}]}";
  String schemaOut = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" +
      "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss\"}}," +
      "{\"name\":\"Product\",\"type\":\"String\"}]}";

  SQLExecEnvironment.getEnvironment()
      .registerTable("ORDERS", new FileEndpoint("dummyFilePathInput", new CSVMessageFormat(schemaIn)))
      .registerTable("SALES", new FileEndpoint("dummyFilePathOutput", "out.tmp", new CSVMessageFormat(schemaOut)))
      .executeSQL(dag, "INSERT INTO SALES SELECT STREAM FLOOR(ROWTIME TO HOUR), SUBSTRING(PRODUCT, 0, 5) " +
      "FROM ORDERS WHERE id > 3");

  dag.validate();
}
 

@Test
public void validatePositiveOiOiOdiamond()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());

  plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout1", intermediateOperator1.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout2", intermediateOperator2.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);

  try {
    plan.validate();
    Assert.assertTrue("OiOiO diamond validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIOIO diamond validation");
  }
}
 

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testModuleTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    throw new RuntimeException("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  planConf.prepareDAG(dag, null, "testApplication");
  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}
 

@Test
public void validatePositiveOiOiOExtendeddiamond()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());

  plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);

  try {
    plan.validate();
    Assert.assertTrue("OiOiO extended diamond validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIOIO extended diamond validation");
  }
}
 

@Test
public void validateOiOImplementation() throws Exception
{
  LogicalPlan lp = new LogicalPlan();
  String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
  lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
  FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
  SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());

  /*
   * This tests make sure that even if the application_window_count is different the endWindow() is called for
   * end_stream
   */
  lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
  StreamMeta stream = lp.addStream("Stream", io.output, go.input);
  StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);

  stream1.setLocality(Locality.THREAD_LOCAL);
  lp.validate();
  StramLocalCluster slc = new StramLocalCluster(lp);
  slc.run();
  Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}
 

@Test
public void testLoadFromPropertiesFile() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testModuleTopology.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    throw new RuntimeException("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false))
      .addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  pb.prepareDAG(dag, null, "testApplication");
  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}
 

@Test
public void validateTestApplication()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  lpc.prepareDAG(dag, new Application(), "TestApp");

  Assert.assertEquals(2, dag.getAllModules().size(), 2);
  Assert.assertEquals(5, dag.getAllOperators().size());
  Assert.assertEquals(4, dag.getAllStreams().size());
  dag.validate();
}
 

@Test
public void testJoinFilter() throws IOException, ClassNotFoundException
{
  LogicalPlan dag = new LogicalPlan();
  String schemaIn0 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" +
      "{\"name\":\"RowTime\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," +
      "{\"name\":\"id\",\"type\":\"Integer\"}," +
      "{\"name\":\"Product\",\"type\":\"String\"}," +
      "{\"name\":\"units\",\"type\":\"Integer\"}]}";
  String schemaIn1 = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" +
      "{\"name\":\"id\",\"type\":\"Integer\"}," +
      "{\"name\":\"Category\",\"type\":\"String\"}]}";
  String schemaOut = "{\"separator\":\",\",\"quoteChar\":\"\\\"\",\"fields\":[" +
      "{\"name\":\"RowTime1\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," +
      "{\"name\":\"RowTime2\",\"type\":\"Date\",\"constraints\":{\"format\":\"dd/MM/yyyy hh:mm:ss Z\"}}," +
      "{\"name\":\"Product\",\"type\":\"String\"}," +
      "{\"name\":\"Category\",\"type\":\"String\"}]}";

  String sql = "INSERT INTO SALES SELECT STREAM A.ROWTIME, FLOOR(A.ROWTIME TO DAY), " +
      "APEXCONCAT('OILPAINT', SUBSTRING(A.PRODUCT, 6, 7)), B.CATEGORY " +
      "FROM ORDERS AS A JOIN CATEGORY AS B ON A.id = B.id AND A.id > 3" +
      "WHERE A.PRODUCT LIKE 'paint%'";

  SQLExecEnvironment.getEnvironment()
      .registerTable("ORDERS", new KafkaEndpoint("localhost:9092", "testdata0", new CSVMessageFormat(schemaIn0)))
      .registerTable("CATEGORY", new KafkaEndpoint("localhost:9092", "testdata1", new CSVMessageFormat(schemaIn1)))
      .registerTable("SALES", new KafkaEndpoint("localhost:9092", "testresult", new CSVMessageFormat(schemaOut)))
      .registerFunction("APEXCONCAT", FileEndpointTest.class, "apex_concat_str")
      .executeSQL(dag, sql);

  dag.validate();
}
 

@Test
public void testSQLWithApexFactory() throws IOException, ClassNotFoundException
{
  File modelFile = new File("src/test/resources/model/model_file_csv.json");
  String model = FileUtils.readFileToString(modelFile);

  LogicalPlan dag = new LogicalPlan();
  SQLExecEnvironment.getEnvironment()
      .withModel(model)
      .executeSQL(dag, "SELECT STREAM ROWTIME, PRODUCT FROM ORDERS");

  dag.validate();
}
 

/**
 * Module and Operator with same name is not allowed in a DAG, to prevent properties
 * conflict.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithOperator2()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
  DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}
 

/**
 * Module and Operator with same name is not allowed in a DAG, to prevent properties
 * conflict.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithOperator1()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
  Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}
 

/**
 * Generate a conflict, Add a top level operator with name "m1_O1",
 * and add a module "m1" which will populate operator "O1", causing name conflict with
 * top level operator.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithExpandedModule()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  DummyInputOperator in = dag.addOperator(componentName("m1", "O1"), new DummyInputOperator());
  Level2ModuleA module = dag.addModule("m1", new Level2ModuleA());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}
 

@Test
public void testModuleExtreme()
{
  StreamingApplication app = new NestedModuleApp();
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  lpc.prepareDAG(dag, app, "ModuleApp");

  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}
 

public StramClient(Configuration conf, LogicalPlan dag) throws Exception
{
  this.conf = conf;
  this.dag = dag;
  dag.validate();
  yarnClient.init(conf);
}
 

private LogicalPlan createDAG(LogicalPlanConfiguration lpc) throws Exception
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.toURI().toString());
  lpc.prepareDAG(dag, null, "testApp");
  dag.validate();
  Assert.assertEquals("", Integer.valueOf(128), dag.getValue(DAG.MASTER_MEMORY_MB));
  Assert.assertEquals("", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS));
  return dag;
}
 

@Test
public void validatePositiveOiOiOdiamondWithCores()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());

  plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);

  plan.setOperatorAttribute(inputOperator, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator1, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator2, OperatorContext.VCORES, 2);
  plan.setOperatorAttribute(intermediateOperator3, OperatorContext.VCORES, 3);
  plan.setOperatorAttribute(intermediateOperator4, OperatorContext.VCORES, 5);
  plan.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  try {
    plan.validate();
    Assert.assertTrue("OiOiO extended diamond validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIOIO extended diamond validation");
  }
  PhysicalPlan physicalPlan = new PhysicalPlan(plan, new TestPlanContext());
  Assert.assertTrue("number of containers", 1 == physicalPlan.getContainers().size());
  Assert.assertTrue("number of vcores " + physicalPlan.getContainers().get(0).getRequiredVCores(), 5 == physicalPlan.getContainers().get(0).getRequiredVCores());
}
 

@Test
public void validatePositiveOiOOptionalInput()
{
  LogicalPlan plan = new LogicalPlan();
  RecoverableInputOperator inputOp1 = plan.addOperator("InputOperator1", new RecoverableInputOperator());
  GenericOperator genOp = plan.addOperator("GenericOperator", new GenericOperator());
  plan.addStream("OiO1", inputOp1.output, genOp.ip1).setLocality(Locality.THREAD_LOCAL);

  try {
    plan.validate();
    Assert.assertTrue("OiO validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OiO Single Connected InputPort");
  }
}
 

public StramClient(Configuration conf, LogicalPlan dag) throws Exception
{
  this.conf = conf;
  this.dag = dag;
  dag.validate();
  yarnClient.init(conf);
}
 

@Test
public void testStaticPartitioning()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  TestGeneratorInputOperator node0 = dag.addOperator("node0", TestGeneratorInputOperator.class);
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  PartitioningTestOperator partitioned = dag.addOperator("partitioned", PartitioningTestOperator.class);
  partitioned.setPartitionCount(partitioned.partitionKeys.length);
  GenericTestOperator singleton1 = dag.addOperator("singleton1", GenericTestOperator.class);
  GenericTestOperator singleton2 = dag.addOperator("singleton2", GenericTestOperator.class);

  dag.addStream("n0.inport1", node0.outport, node1.inport1);
  dag.addStream("n1.outport1", node1.outport1, partitioned.inport1, partitioned.inportWithCodec);
  dag.addStream("mergeStream", partitioned.outport1, singleton1.inport1, singleton2.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);

  OperatorMeta partitionedMeta = dag.getMeta(partitioned);

  dag.validate();

  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());

  Assert.assertEquals("number of containers", 2, plan.getContainers().size());
  Assert.assertNotNull("partition map", partitioned.partitions);
  Assert.assertEquals("partition map " + partitioned.partitions, 3, partitioned.partitions.size());

  List<PTOperator> n2Instances = plan.getOperators(partitionedMeta);
  Assert.assertEquals("partition instances " + n2Instances, partitioned.partitionKeys.length, n2Instances.size());
  for (int i = 0; i < n2Instances.size(); i++) {
    PTOperator po = n2Instances.get(i);
    Map<String, PTInput> inputsMap = new HashMap<>();
    for (PTInput input: po.getInputs()) {
      inputsMap.put(input.portName, input);
      Assert.assertEquals("partitions " + input, Sets.newHashSet(partitioned.partitionKeys[i]), input.partitions.partitions);
      //Assert.assertEquals("codec " + input.logicalStream, PartitioningTestStreamCodec.class, input.logicalStream.getCodecClass());
    }
    Assert.assertEquals("number inputs " + inputsMap, Sets.newHashSet(PartitioningTestOperator.IPORT1, PartitioningTestOperator.INPORT_WITH_CODEC), inputsMap.keySet());
  }

  Collection<PTOperator> unifiers = plan.getMergeOperators(partitionedMeta);
  Assert.assertEquals("number unifiers " + partitionedMeta, 0, unifiers.size());
}