com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta Java Examples

@Test
public void validateOiOImplementation() throws Exception
{
  LogicalPlan lp = new LogicalPlan();
  String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
  lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
  FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
  SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());

  /*
   * This tests make sure that even if the application_window_count is different the endWindow() is called for
   * end_stream
   */
  lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
  StreamMeta stream = lp.addStream("Stream", io.output, go.input);
  StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);

  stream1.setLocality(Locality.THREAD_LOCAL);
  lp.validate();
  StramLocalCluster slc = new StramLocalCluster(lp);
  slc.run();
  Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}
 

public static PTOperator createSlidingUnifier(StreamMeta streamMeta, PhysicalPlan plan, int
    operatorApplicationWindowCount, int slidingWindowCount)
{
  int gcd = IntMath.gcd(operatorApplicationWindowCount, slidingWindowCount);
  OperatorMeta um = streamMeta.getSource()
      .getSlidingUnifier(operatorApplicationWindowCount / gcd, gcd, slidingWindowCount / gcd);
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }
  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan)
{
  OperatorMeta um = streamMeta.getSource().getUnifierMeta();
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }

  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan)
{
  OperatorMeta um = streamMeta.getSource().getUnifierMeta();
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }

  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public static PTOperator createSlidingUnifier(StreamMeta streamMeta, PhysicalPlan plan, int
    operatorApplicationWindowCount, int slidingWindowCount)
{
  int gcd = IntMath.gcd(operatorApplicationWindowCount, slidingWindowCount);
  OperatorMeta um = streamMeta.getSource()
      .getSlidingUnifier(operatorApplicationWindowCount / gcd, gcd, slidingWindowCount / gcd);
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }
  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

@Test
public void testDeleteOperator()
{
  TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  dag.addStream("s0", input.outport, o1.inport1);
  StreamMeta s1 = dag.addStream("s1", o1.outport1, o2.inport1);
  dag.validate();
  Assert.assertEquals("", 3, dag.getAllOperators().size());

  dag.removeOperator(o2);
  s1.remove();
  dag.validate();
  Assert.assertEquals("", 2, dag.getAllOperators().size());
}
 

@Test
public void testLocalityValidation()
{
  TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  StreamMeta s1 = dag.addStream("input1.outport", input1.outport, o1.inport1).setLocality(Locality.THREAD_LOCAL);
  dag.validate();

  TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
  dag.addStream("input2.outport", input2.outport, o1.inport2);

  try {
    dag.validate();
    Assert.fail("Exception expected for " + o1);
  } catch (ValidationException ve) {
    Assert.assertThat("", ve.getMessage(), RegexMatcher.matches("Locality THREAD_LOCAL invalid for operator .* with multiple input streams .*"));
  }

  s1.setLocality(null);
  dag.validate();
}
 

@Test
public void testLoadFromPropertiesFileWithLegacyPrefix() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testTopologyLegacyPrefix.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  dag.validate();

  assertEquals("number of operators", 2, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("s1");
  assertNotNull(s1);
  assertTrue("s1 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta o2m = dag.getOperatorMeta("o2");
  assertEquals(GenericTestOperator.class, o2m.getOperator().getClass());
  GenericTestOperator o2 = (GenericTestOperator)o2m.getOperator();
  assertEquals("myStringProperty " + o2, "myStringPropertyValue", o2.getMyStringProperty());
}
 

private void printTopology(OperatorMeta operator, DAG tplg, int level)
{
  String prefix = "";
  if (level > 0) {
    prefix = StringUtils.repeat(" ", 20 * (level - 1)) + "   |" + StringUtils.repeat("-", 17);
  }
  logger.debug(prefix + operator.getName());
  for (StreamMeta downStream : operator.getOutputStreams().values()) {
    if (!downStream.getSinks().isEmpty()) {
      for (LogicalPlan.InputPortMeta targetNode : downStream.getSinks()) {
        printTopology(targetNode.getOperatorMeta(), tplg, level + 1);
      }
    }
  }
}
 

public void dfs(OperatorMeta o, List<OperatorMeta> visited) {
  visited.add(o);

  for (Entry<OutputPortMeta, StreamMeta> opm : o.getOutputStreams().entrySet()) {
    // Samoa won't allow one output port to multiple input port kind of streams
    OperatorMeta downStreamOp = opm.getValue().getSinks().get(0).getOperatorWrapper();
    if (visited.contains(downStreamOp)) {
      loopStreams.add(opm.getValue());
    } else {
      List<OperatorMeta> v2 = Lists.newArrayList();
      v2.addAll(visited);
      dfs(downStreamOp, v2);
    }
  }
}
 

/**
 * Remove the named stream. Ignored when stream does not exist.
 * @param streamName
 */
public void removeStream(String streamName)
{
  StreamMeta sm = logicalPlan.getStream(streamName);
  if (sm == null) {
    return;
  }

  if (physicalPlan != null) {
    // associated operators will redeploy
    physicalPlan.removeLogicalStream(sm);
  }
  // remove from logical plan
  sm.remove();
}
 

public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm != null) {
    if (sm.getSource().getOperatorMeta().getMeta(source) != sm.getSource()) {
      throw new AssertionError(String.format("Stream %s already connected to %s", sm, sm.getSource()));
    }
  } else {
    // fails on duplicate stream name
    @SuppressWarnings("unchecked")
    StreamMeta newStream = logicalPlan.addStream(id, source);
    sm = newStream;
  }
  return addSinks(id, sinks);
}
 

@Test
public void testTupleClassAttr() throws Exception
{
  String resourcePath = "/schemaTestTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  OperatorMeta operator1 = dag.getOperatorMeta("operator1");
  assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
  }
}
 

/**
 * Constructor
 *
 * @param portName
 * @param logicalStream
 * @param source
 */
protected PTOutput(String portName, StreamMeta logicalStream, PTOperator source)
{
  this.logicalStream = logicalStream;
  this.source = source;
  this.portName = portName;
  this.sinks = new ArrayList<>();
}
 

/**
 *
 * @param portName
 * @param logicalStream
 * @param target
 * @param partitions
 * @param source
 */
protected PTInput(String portName, StreamMeta logicalStream, PTOperator target, PartitionKeys partitions, PTOutput source, boolean delay)
{
  this.logicalStream = logicalStream;
  this.target = target;
  this.partitions = partitions;
  this.source = source;
  this.portName = portName;
  this.source.sinks.add(this);
  this.delay = delay;
}
 

private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
 

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

/**
 *
 * @param portName
 * @param logicalStream
 * @param target
 * @param partitions
 * @param source
 */
protected PTInput(String portName, StreamMeta logicalStream, PTOperator target, PartitionKeys partitions, PTOutput source, boolean delay)
{
  this.logicalStream = logicalStream;
  this.target = target;
  this.partitions = partitions;
  this.source = source;
  this.portName = portName;
  this.source.sinks.add(this);
  this.delay = delay;
}
 

public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input)
{
  InputPortMeta inputPortMeta;
  PTOperator inputTarget = input.target;
  StreamMeta streamMeta = input.logicalStream;
  if (!inputTarget.isUnifier()) {
    inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
  } else {
    PTOperator destTarget = getIdentifyingOperator(inputTarget);
    inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
  }
  return inputPortMeta;
}
 

private void setStreamConfiguration(LogicalPlan dag, List<AppConf> appConfs, String appAlias)
{
  for (StreamMeta sm : dag.getAllStreams()) {
    List<StreamConf> smConfs = getMatchingChildConf(appConfs, sm.getName(), StramElement.STREAM);
    for (StreamConf smConf : smConfs) {
      DAG.Locality locality = smConf.getLocality();
      if (locality != null) {
        sm.setLocality(locality);
        break;
      }
    }
  }
}
 

private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
 

/**
 * Constructor
 *
 * @param portName
 * @param logicalStream
 * @param source
 */
protected PTOutput(String portName, StreamMeta logicalStream, PTOperator source)
{
  this.logicalStream = logicalStream;
  this.source = source;
  this.portName = portName;
  this.sinks = new ArrayList<>();
}
 

public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm != null) {
    if (sm.getSource().getOperatorMeta().getMeta(source) != sm.getSource()) {
      throw new AssertionError(String.format("Stream %s already connected to %s", sm, sm.getSource()));
    }
  } else {
    // fails on duplicate stream name
    @SuppressWarnings("unchecked")
    StreamMeta newStream = logicalPlan.addStream(id, source);
    sm = newStream;
  }
  return addSinks(id, sinks);
}
 

/**
 * Remove the named stream. Ignored when stream does not exist.
 * @param streamName
 */
public void removeStream(String streamName)
{
  StreamMeta sm = logicalPlan.getStream(streamName);
  if (sm == null) {
    return;
  }

  if (physicalPlan != null) {
    // associated operators will redeploy
    physicalPlan.removeLogicalStream(sm);
  }
  // remove from logical plan
  sm.remove();
}
 

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input)
{
  InputPortMeta inputPortMeta;
  PTOperator inputTarget = input.target;
  StreamMeta streamMeta = input.logicalStream;
  if (!inputTarget.isUnifier()) {
    inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
  } else {
    PTOperator destTarget = getIdentifyingOperator(inputTarget);
    inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
  }
  return inputPortMeta;
}
 

private void setStreamConfiguration(LogicalPlan dag, List<AppConf> appConfs, String appAlias)
{
  for (StreamMeta sm : dag.getAllStreams()) {
    List<StreamConf> smConfs = getMatchingChildConf(appConfs, sm.getName(), StramElement.STREAM);
    for (StreamConf smConf : smConfs) {
      DAG.Locality locality = smConf.getLocality();
      if (locality != null) {
        sm.setLocality(locality);
        break;
      }
    }
  }
}
 

/**
 * Add logical operator to the plan. Assumes that upstream operators have been added before.
 * @param om
 */
public final void addLogicalOperator(OperatorMeta om)
{
  PMapping pnodes = new PMapping(om);
  String host = pnodes.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  localityPrefs.add(pnodes, host);

  PMapping upstreamPartitioned = null;

  for (Map.Entry<LogicalPlan.InputPortMeta, StreamMeta> e : om.getInputStreams().entrySet()) {
    if (e.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
      continue;
    }
    PMapping m = logicalToPTOperator.get(e.getValue().getSource().getOperatorMeta());
    if (e.getKey().getValue(PortContext.PARTITION_PARALLEL).equals(true)) {
      // operator partitioned with upstream
      if (upstreamPartitioned != null) {
        // need to have common root
        if (!upstreamPartitioned.parallelPartitions.contains(m.logicalOperator) && upstreamPartitioned != m) {
          String msg = String.format("operator cannot extend multiple partitions (%s and %s)", upstreamPartitioned.logicalOperator, m.logicalOperator);
          throw new AssertionError(msg);
        }
      }
      m.parallelPartitions.add(pnodes.logicalOperator);
      pnodes.parallelPartitions = m.parallelPartitions;
      upstreamPartitioned = m;
    }

    if (Locality.CONTAINER_LOCAL == e.getValue().getLocality() || Locality.THREAD_LOCAL == e.getValue().getLocality()) {
      inlinePrefs.setLocal(m, pnodes);
    } else if (Locality.NODE_LOCAL == e.getValue().getLocality()) {
      localityPrefs.setLocal(m, pnodes);
    }
  }

  //
  // create operator instances
  //
  this.logicalToPTOperator.put(om, pnodes);
  if (upstreamPartitioned != null) {
    // parallel partition
    //LOG.debug("Operator {} should be partitioned to {} partitions", pnodes.logicalOperator.getName(), upstreamPartitioned.partitions.size());
    initPartitioning(pnodes, upstreamPartitioned.partitions.size());
  } else {
    initPartitioning(pnodes, 0);
  }
  updateStreamMappings(pnodes);
}