Java Code Examples for com.datatorrent.api.Operator#InputPort

public static void initCommonConfigsForAllTypesOfTests() throws Exception
{
  KuduClientTestCommons.buildSchemaForUnitTestsTable();
  partitioningContext = new Partitioner.PartitioningContext()
  {
    @Override
    public int getParallelPartitionCount()
    {
      return numberOfKuduInputOperatorPartitions;
    }

    @Override
    public List<Operator.InputPort<?>> getInputPorts()
    {
      return null;
    }
  };
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

@Override
@SuppressWarnings("unchecked")
public  <O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort, Operator.OutputPort<O> outputPort, Option... opts)
{

  checkArguments(op, inputPort, outputPort);

  DagMeta.NodeMeta nm = null;

  if (lastBrick == null) {
    nm = graph.addNode(op, null, null, inputPort, opts);
  } else {
    nm = graph.addNode(op, lastBrick.nodeMeta, lastBrick.lastOutput, inputPort, opts);
  }

  Brick<O> newBrick = new Brick<>();
  newBrick.nodeMeta = nm;
  newBrick.setLastOutput(outputPort);
  if (lastBrick != null) {
    newBrick.lastStream = Pair.<Operator.OutputPort, Operator.InputPort>of(lastBrick.lastOutput, inputPort);
  }

  if (this.getClass() == ApexStreamImpl.class || this.getClass() == ApexWindowedStreamImpl.class) {
    return (STREAM)newStream(this.graph, newBrick);
  } else {
    try {
      return (STREAM)this.getClass().getConstructor(ApexStreamImpl.class).newInstance(newStream(this.graph, newBrick));
    } catch (Exception e) {
      throw new RuntimeException("You have to override the default constructor with ApexStreamImpl as default parameter", e);
    }
  }

}
 

private void checkArguments(Operator op, Operator.InputPort inputPort, Operator.OutputPort outputPort)
{
  if (op == null) {
    throw new IllegalArgumentException("Operator can not be null");
  }

  boolean foundInput = inputPort == null;
  boolean foundOutput = outputPort == null;
  for (Field f : op.getClass().getFields()) {
    int modifiers = f.getModifiers();
    if (!Modifier.isPublic(modifiers) || !Modifier.isTransient(modifiers)) {
      continue;
    }
    Object obj = null;
    try {
      obj = f.get(op);
    } catch (IllegalAccessException e) {
      // NonAccessible field is not a valid port object
    }
    if (obj == outputPort) {
      foundOutput = true;
    }
    if (obj == inputPort) {
      foundInput = true;
    }
  }
  if (!foundInput || !foundOutput) {
    throw new IllegalArgumentException("Input port " + inputPort + " and/or Output port " + outputPort + " is/are not owned by Operator " + op);
  }

}
 

private void checkPresentStreamCodec(LogicalPlan.OperatorMeta operatorMeta, Operator.InputPort<?> inputPort,
    Map<Integer, StreamCodec<?>> streamCodecs,
    String id, PhysicalPlan plan)
{
  StreamCodec<?> streamCodec = operatorMeta.getMeta(inputPort).getStreamCodec();
  Assert.assertTrue("stream codec identifier not present" + id, isStrCodecPresent(streamCodec, plan));
  Integer streamCodecIdentifier = plan.getStreamCodecIdentifier(streamCodec);
  checkPresentStreamCodecInfo(streamCodecs, id, streamCodecIdentifier, streamCodec);
}
 

public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias)
{
  if (PubSubHelper.isGatewayConfigured(dag)) {
    URI uri = PubSubHelper.getURI(dag);

    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());

    Map<String, String> conversionMap = Maps.newHashMap();
    conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);

    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    snapshotServer.setTableFieldToMapField(conversionMap);

    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.setUri(uri);
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);

    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setUri(uri);
    Operator.InputPort<String> queryResultPort = wsResult.input;

    dag.addStream("MapProvider", topCount, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
  } else {
    ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
    operator.setStringFormat(operatorName + ": %s");

    dag.addStream("MapProvider", topCount, operator.input);
  }
}
 

public RelInfo(String relName, List<Operator.InputPort> inputPorts, Operator operator, Operator.OutputPort outPort, Class clazz)
{
  this.inputPorts = inputPorts;
  this.operator = operator;
  this.outPort = outPort;
  this.clazz = clazz;
  this.relName = relName;
  this.outRelDataType = null;
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1, Operator.InputPort<? super T> sink2)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[] {sink1, sink2};
  return addStream(id, source, ports);
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[]{sink1};
  return addStream(id, source, ports);
}
 

@Override
@SuppressWarnings("unchecked")
public ApexStream<T> with(DAG.Locality locality)
{
  if (lastBrick.lastStream != null) {
    for (DagMeta.NodeMeta parent : lastBrick.nodeMeta.getParent()) {
      Pair<List<Operator.InputPort>, DAG.Locality> p = parent.getNodeStreams().get(lastBrick.lastStream.getLeft());
      if (p != null) {
        p.setValue(locality);
      }
    }
  }
  return this;
}
 

/**
 * Populates the DAG with operators and connecting streams
 *
 * @param dag The directed acyclic graph of operators to populate
 * @param conf The configuration
 */
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  // create operators
  LineReader lineReader            = dag.addOperator("lineReader", new LineReader());
  WordReader wordReader            = dag.addOperator("wordReader", new WordReader());
  WindowWordCount windowWordCount  = dag.addOperator("windowWordCount", new WindowWordCount());
  FileWordCount fileWordCount      = dag.addOperator("fileWordCount", new FileWordCount());
  WordCountWriter wcWriter         = dag.addOperator("wcWriter", new WordCountWriter());
  ConsoleOutputOperator console    = dag.addOperator("console", new ConsoleOutputOperator());
  console.setStringFormat("wordCount: %s");

  // create streams

  dag.addStream("lines",   lineReader.output,  wordReader.input);
  dag.addStream("control", lineReader.control, fileWordCount.control);
  dag.addStream("words",   wordReader.output,  windowWordCount.input);
  dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
  dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);

  if (PubSubHelper.isGatewayConfigured(dag)) {        // add query support
    URI uri = PubSubHelper.getURI(dag);

    AppDataSnapshotServerMap snapshotServerFile
        = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
    AppDataSnapshotServerMap snapshotServerGlobal
        = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());

    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
    snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
    snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);

    PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
    PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
    wsQueryFile.setUri(uri);
    wsQueryGlobal.setUri(uri);

    snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
    snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);

    PubSubWebSocketAppDataResult wsResultFile
        = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
    PubSubWebSocketAppDataResult wsResultGlobal
        = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
    wsResultFile.setUri(uri);
    wsResultGlobal.setUri(uri);

    Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
    Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;

    dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
    dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);

    dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
    dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
  } else {
    //throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
    dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
  }

  LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
  LOG.info("Returning from populateDAG");
}
 

/**
 * This is the main method in this relational node visitor which traverses the relational algebra in reverse direction
 * and populate the given underlying DAG object.
 *
 * @param relNode RelNode which needs to be traversed.
 *
 * @return RelInfo representing information of current stage
 * @throws Exception
 */
public final RelInfo traverse(RelNode relNode) throws Exception
{
  List<RelInfo> inputStreams = new ArrayList<>();
  for (RelNode input : relNode.getInputs()) {
    inputStreams.add(traverse(input));
  }

  ApexRelNode.RelContext relContext = new ApexRelNode.RelContext(dag, typeFactory, tupleSchemaRegistry);

  RelInfo currentNodeRelInfo;
  ApexRelNode apexRelNode = ApexRelNode.relNodeMapping.get(relNode.getClass());
  if (apexRelNode == null) {
    throw new UnsupportedOperationException("RelNode " + relNode.getRelTypeName() + " is not supported.");
  }
  currentNodeRelInfo = apexRelNode.visit(relContext, relNode, inputStreams);

  if (currentNodeRelInfo != null && inputStreams.size() != 0) {
    for (int i = 0; i < inputStreams.size(); i++) {
      RelInfo inputStream = inputStreams.get(i);
      Operator.OutputPort outputPort = inputStream.getOutPort();
      Operator.InputPort inputPort = currentNodeRelInfo.getInputPorts().get(i);

      String streamName = OperatorUtils.getUniqueStreamName(inputStream.getRelName(),
          currentNodeRelInfo.getRelName());
      Class schema;
      if (inputStream.getOutRelDataType() != null) {
        schema = TupleSchemaRegistry.getSchemaForRelDataType(tupleSchemaRegistry, streamName,
            inputStream.getOutRelDataType());
      } else if (inputStream.getClazz() != null) {
        schema = inputStream.getClazz();
      } else {
        throw new RuntimeException("Unexpected condition reached.");
      }
      dag.setOutputPortAttribute(outputPort, Context.PortContext.TUPLE_CLASS, schema);
      dag.setInputPortAttribute(inputPort, Context.PortContext.TUPLE_CLASS, schema);
      dag.addStream(streamName, outputPort, inputPort);
    }
  }

  if (currentNodeRelInfo.getOutPort() == null) {
    // End of the pipeline.
    String schemaJar = tupleSchemaRegistry.generateCommonJar();

    String jars = dag.getAttributes().get(Context.DAGContext.LIBRARY_JARS);
    dag.setAttribute(Context.DAGContext.LIBRARY_JARS,
        ((jars != null) && (jars.length() != 0)) ? jars + "," + schemaJar : schemaJar);
  }

  return currentNodeRelInfo;
}
 

@Override
public InputPortMeta getMeta(Operator.InputPort<?> port)
{
  return getPortMapping().inPortMap.get(port);
}
 

public void setLastStream(Pair<Operator.OutputPort, Operator.InputPort> lastStream)
{
  this.lastStream = lastStream;
}
 

public StreamEndpoint(Operator.InputPort port, Map<String, Class> fieldMapping)
{
  this.inputPort = port;
  this.fieldMapping = fieldMapping;
}
 

@Override
public <O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts)
{
  return (STREAM)addOperator(op, inputPort, null, opts);
}
 

public Map<Operator.OutputPort, Pair<List<Operator.InputPort>, DAG.Locality>> getNodeStreams()
{
  return nodeStreams;
}
 

/**
 * Extend the dag by adding one operator<br>
 * @param op Operator added to the stream
 * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream
 * @param outputPort OutputPort of the operator will be connected to next operator
 * @param <O> type of the output
 * @return new stream of type O
 */
<O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort,  Operator.OutputPort<O> outputPort, Option... opts);
 

/**
 * Extend the dag by adding one end operator<br>
 * @param op Operator added to the stream
 * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream
 * @param <O> type of the output
 * @return new stream of type O
 */
<O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts);
 

void addInputPort(Operator.InputPort<?> port, Field field, InputPortFieldAnnotation portAnnotation, AppData.QueryPort adqAnnotation);