com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta Java Examples

The following examples show how to use com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PhysicalPlan.java    From Bats with Apache License 2.0 5 votes vote down vote up 
private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
Example #2
Source File: PhysicalPlan.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
Example #3
Source File: LogicalPlanConfigurationTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
private void simpleNameOutputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
{
  OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);

  for (OutputPortMeta outputPortMeta: operatorMeta.getOutputStreams().keySet()) {
    if (set) {
      Assert.assertEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
    } else {
      Assert.assertNotEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
    }
  }
}
Example #4
Source File: ApexTask.java    From incubator-samoa with Apache License 2.0 5 votes vote down vote up 
public void dfs(OperatorMeta o, List<OperatorMeta> visited) {
  visited.add(o);

  for (Entry<OutputPortMeta, StreamMeta> opm : o.getOutputStreams().entrySet()) {
    // Samoa won't allow one output port to multiple input port kind of streams
    OperatorMeta downStreamOp = opm.getValue().getSinks().get(0).getOperatorWrapper();
    if (visited.contains(downStreamOp)) {
      loopStreams.add(opm.getValue());
    } else {
      List<OperatorMeta> v2 = Lists.newArrayList();
      v2.addAll(visited);
      dfs(downStreamOp, v2);
    }
  }
}
Example #5
Source File: LogicalPlanSerializer.java    From Bats with Apache License 2.0 4 votes vote down vote up 
public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
Example #6
Source File: LogicalPlanSerializer.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
Example #7
Source File: LogicalPlanConfigurationTest.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
  Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
  stringCodecsMap.put(Integer.class, Integer2String.class);
  assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
  assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta input = dag.getOperatorMeta("inputOperator");
  TestStatsListener tsl = new TestStatsListener();
  tsl.setIntProp(222);
  List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
  assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
  for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
    assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
  }

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
  Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }
  Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
  Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}