com.datatorrent.api.Operator.InputPort Java Examples

@Override
    public Collection<com.datatorrent.api.Partitioner.Partition<T>> definePartitions(
        Collection<com.datatorrent.api.Partitioner.Partition<T>> partitions,
        com.datatorrent.api.Partitioner.PartitioningContext context) {
      //Get a partition
      DefaultPartition<T> partition = (DefaultPartition<T>) partitions.iterator().next();
      Collection<Partition<T>> newPartitions;

      // first call to define partitions
      newPartitions = Lists.newArrayList();

      for (int partitionCounter = 0; partitionCounter < newPartitionCount; partitionCounter++) {
        newPartitions.add(new DefaultPartition<T>(partition.getPartitionedInstance()));
      }
      List<InputPort<?>> inputPortList = context.getInputPorts();
//      if (inputPortList != null && !inputPortList.isEmpty()) {
//        DefaultPartition.assignPartitionKeys(newPartitions, inputPortList.iterator().next());
//      }

      return newPartitions;
    }
 

/**
 * Get console for output operator.
 * @param name  Operator name
 * @param dag   Application DAG graph.
 * @return input port for console output.
 */
public InputPort<Object> getConsole(String name, /*String nodeName,*/ DAG dag, String prefix)
{
  // hack to output to HTTP based on actual environment
      /*
   String serverAddr = System.getenv("MALHAR_AJAXSERVER_ADDRESS");
   if (serverAddr != null) {
   HttpOutputOperator<Object> oper = dag.addOperator(name, new HttpOutputOperator<Object>());
   oper.setResourceURL(URI.create("http://" + serverAddr + "/channel/" + nodeName));
   return oper.input;
   }
   */

  ConsoleOutputOperator oper = dag.addOperator(name, ConsoleOutputOperator.class);
  oper.setStringFormat(prefix + ": %s");
  return oper.input;
}
 

/**
 * Go over each Proxy port and find out the actual port connected to the ProxyPort
 * and update StreamMeta.
 */
private void resolvePorts()
{
  if (proxySource != null && proxySource instanceof ProxyOutputPort) {
    OutputPort<?> outputPort = proxySource;
    while (outputPort instanceof ProxyOutputPort) {
      outputPort = ((ProxyOutputPort<?>)outputPort).get();
    }
    setSource(outputPort);
  }

  for (InputPort<?> inputPort : proxySinks) {
    while (inputPort instanceof ProxyInputPort) {
      inputPort = ((ProxyInputPort<?>)inputPort).get();
    }
    addSink(inputPort);
  }

  proxySource = null;
  proxySinks.clear();
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

@Override
public void connectInputPort(String port, final SweepableReservoir reservoir)
{
  if (reservoir == null) {
    throw new IllegalArgumentException("Reservoir cannot be null for port '" + port + "' on operator '" + operator + "'");
  }

  InputPort<Object> inputPort = getInputPort(port);
  if (inputPort == null) {
    throw new IllegalArgumentException("Port '" + port + "' does not exist on operator '" + operator + "'");
  }

  if (inputs.containsKey(port)) {
    deferredInputConnections.add(new DeferredInputConnection(port, reservoir));
  } else {
    inputPort.setConnected(true);
    inputs.put(port, reservoir);
    reservoir.setSink(inputPort.getSink());
  }
}
 

@Override
public StreamMeta addSink(Operator.InputPort<?> port)
{
  if (port instanceof ProxyInputPort) {
    proxySinks.add(port);
    return this;
  }
  InputPortMeta portMeta = assertGetPortMeta(port);
  OperatorMeta om = portMeta.getOperatorMeta();
  String portName = portMeta.getPortName();
  if (om.inputStreams.containsKey(portMeta)) {
    throw new IllegalArgumentException(String.format("Port %s already connected to stream %s", portName, om.inputStreams.get(portMeta)));
  }

  /*
  finalizeValidate(portMeta);
  */

  sinks.add(portMeta);
  om.inputStreams.put(portMeta, this);
  rootOperators.remove(portMeta.operatorMeta);
  if (this.source != null && !(this.source.getOperatorMeta().getOperator() instanceof Operator.DelayOperator)) {
    leafOperators.remove(this.source.getOperatorMeta());
  }
  return this;
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(@Nonnull String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T>... sinks)
{
  StreamMeta s = addStream(id);
  s.setSource(source);
  for (Operator.InputPort<?> sink : sinks) {
    s.addSink(sink);
  }
  return s;
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
{
  LogicalPlan subDag = moduleMeta.getDag();
  String subDAGName = moduleMeta.getName();
  String name;
  for (OperatorMeta operatorMeta : subDag.getAllOperators()) {
    name = subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
    Operator op = this.addOperator(name, operatorMeta.getOperator());
    OperatorMeta operatorMetaNew = this.getMeta(op);
    operatorMetaNew.copyAttributesFrom(operatorMeta);
    operatorMetaNew.setModuleName(operatorMeta.getModuleName() == null ? subDAGName :
        subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getModuleName());
  }

  for (StreamMeta streamMeta : subDag.getAllStreams()) {
    OutputPortMeta sourceMeta = streamMeta.getSource();
    List<InputPort<?>> ports = new LinkedList<>();
    for (InputPortMeta inputPortMeta : streamMeta.getSinks()) {
      ports.add(inputPortMeta.getPort());
    }
    InputPort[] inputPorts = ports.toArray(new InputPort[]{});

    name = subDAGName + MODULE_NAMESPACE_SEPARATOR + streamMeta.getName();
    StreamMeta streamMetaNew = this.addStream(name, sourceMeta.getPort(), inputPorts);
    streamMetaNew.setLocality(streamMeta.getLocality());
  }
}
 

public DefaultPartition(T partitionable, Map<InputPort<?>, PartitionKeys> partitionKeys, int loadIndicator, BatchedOperatorStats stats)
{
  this.partitionable = partitionable;
  this.partitionKeys = new PartitionPortMap();
  this.partitionKeys.putAll(partitionKeys);
  this.partitionKeys.modified = false;
  this.loadIndicator = loadIndicator;
  this.stats = stats;
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[]{sink1};
  return addStream(id, source, ports);
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1, Operator.InputPort<? super T> sink2)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[] {sink1, sink2};
  return addStream(id, source, ports);
}
 

private InputPortMeta assertGetPortMeta(Operator.InputPort<?> port)
{
  for (OperatorMeta o : getAllOperators()) {
    InputPortMeta opm = o.getPortMapping().inPortMap.get(port);
    if (opm != null) {
      return opm;
    }
  }
  throw new IllegalArgumentException("Port is not associated to any operator in the DAG: " + port);
}
 

@Override
public void addInputPort(Operator.InputPort<?> port, Field field, InputPortFieldAnnotation portAnnotation, AppData.QueryPort adqAnnotation)
{
  if (!inputPorts.containsKey(field.getName())) {
    inputPorts.put(field.getName(), new PortContextPair<InputPort<?>>(port));
  }
}
 

public static void describe(GenericOperator operator, OperatorDescriptor descriptor)
{
  for (Class<?> c = operator.getClass(); c != Object.class; c = c.getSuperclass()) {
    Field[] fields = c.getDeclaredFields();
    for (Field field : fields) {
      field.setAccessible(true);
      InputPortFieldAnnotation inputAnnotation = field.getAnnotation(InputPortFieldAnnotation.class);
      OutputPortFieldAnnotation outputAnnotation = field.getAnnotation(OutputPortFieldAnnotation.class);
      AppData.QueryPort adqAnnotation = field.getAnnotation(AppData.QueryPort.class);
      AppData.ResultPort adrAnnotation = field.getAnnotation(AppData.ResultPort.class);

      try {
        Object portObject = field.get(operator);

        if (portObject instanceof InputPort) {
          descriptor.addInputPort((Operator.InputPort<?>)portObject, field, inputAnnotation, adqAnnotation);
        } else {
          if (inputAnnotation != null) {
            throw new IllegalArgumentException("port is not of type " + InputPort.class.getName() + ": " + field);
          }
        }

        if (portObject instanceof OutputPort) {
          descriptor.addOutputPort((Operator.OutputPort<?>)portObject, field, outputAnnotation, adrAnnotation);
        } else {
          if (outputAnnotation != null) {
            throw new IllegalArgumentException("port is not of type " + OutputPort.class.getName() + ": " + field);
          }
        }
      } catch (IllegalAccessException e) {
        throw new RuntimeException(e);
      }
    }
  }
}
 

@Override
public List<InputPort<?>> getInputPorts()
{
  if (inputPorts == null) {
    inputPorts = getInputPortList(om.logicalOperator);
  }
  return inputPorts;
}
 

RepartitionContext(Partitioner<Operator> partitioner, PMapping currentMapping, int partitionCount)
{
  super(currentMapping, partitionCount);
  this.operators = currentMapping.partitions;
  this.currentPartitions = new ArrayList<>(operators.size());
  this.currentPartitionMap = Maps.newHashMapWithExpectedSize(operators.size());
  this.operatorIdToPartition = Maps.newHashMapWithExpectedSize(operators.size());

  // collect current partitions with committed operator state
  // those will be needed by the partitioner for split/merge
  for (PTOperator pOperator : operators) {
    Map<InputPort<?>, PartitionKeys> pks = pOperator.getPartitionKeys();
    if (pks == null) {
      throw new AssertionError("Null partition: " + pOperator);
    }

    // if partitions checkpoint at different windows, processing for new or modified
    // partitions will start from earliest checkpoint found (at least once semantics)
    if (minCheckpoint == null) {
      minCheckpoint = pOperator.recoveryCheckpoint;
    } else if (minCheckpoint.windowId > pOperator.recoveryCheckpoint.windowId) {
      minCheckpoint = pOperator.recoveryCheckpoint;
    }

    Operator partitionedOperator = loadOperator(pOperator);
    DefaultPartition<Operator> partition = new DefaultPartition<>(partitionedOperator, pks, pOperator.loadIndicator, pOperator.stats);
    currentPartitions.add(partition);
    currentPartitionMap.put(partition, pOperator);
    LOG.debug("partition load: {} {} {}", pOperator, partition.getPartitionKeys(), partition.getLoad());
    operatorIdToPartition.put(pOperator.getId(), partition);
  }

  newPartitions = partitioner.definePartitions(new ArrayList<Partition<Operator>>(currentPartitions), this);
}
 

private List<InputPort<?>> getInputPortList(LogicalPlan.OperatorMeta operatorMeta)
{
  List<InputPort<?>> inputPortList = Lists.newArrayList();

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    inputPortList.add(inputPortMeta.getPort());
  }

  return inputPortList;
}
 

public Map<InputPort<?>, PartitionKeys> getPartitionKeys()
{
  Map<InputPort<?>, PartitionKeys> pkeys = null;
  if (partitionKeys != null) {
    pkeys = Maps.newHashMapWithExpectedSize(partitionKeys.size());
    for (Map.Entry<InputPortMeta, PartitionKeys> e : partitionKeys.entrySet()) {
      pkeys.put(e.getKey().getPort(), e.getValue());
    }
  }
  return pkeys;
}
 

public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm != null) {
    if (sm.getSource().getOperatorMeta().getMeta(source) != sm.getSource()) {
      throw new AssertionError(String.format("Stream %s already connected to %s", sm, sm.getSource()));
    }
  } else {
    // fails on duplicate stream name
    @SuppressWarnings("unchecked")
    StreamMeta newStream = logicalPlan.addStream(id, source);
    sm = newStream;
  }
  return addSinks(id, sinks);
}
 

private InputPort<?> getInputPort(String operName, String portName)
{
  OperatorMeta om = assertGetOperator(operName);
  Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
  Operators.describe(om.getOperator(), portMap);
  PortContextPair<InputPort<?>> port = portMap.inputPorts.get(portName);
  if (port == null) {
    throw new AssertionError(String.format("Invalid port %s (%s)", portName, om));
  }
  return port.component;
}
 

@Override
public Collection<Partition<T>> definePartitions(Collection<Partition<T>> partitions, PartitioningContext context)
{
  final int newPartitionCount = DefaultPartition.getRequiredPartitionCount(context, this.partitionCount);
  logger.debug("define partitions, partitionCount current {} requested {}", partitions.size(), newPartitionCount);

  //Get a partition
  DefaultPartition<T> partition = (DefaultPartition<T>)partitions.iterator().next();
  Collection<Partition<T>> newPartitions;

  if (partitions.iterator().next().getStats() == null) {
    // first call to define partitions
    newPartitions = Lists.newArrayList();

    for (int partitionCounter = 0; partitionCounter < newPartitionCount; partitionCounter++) {
      newPartitions.add(new DefaultPartition<>(partition.getPartitionedInstance()));
    }

    // partition the stream that was first connected in the DAG and send full data to remaining input ports
    // this gives control over which stream to partition under default partitioning to the DAG writer
    List<InputPort<?>> inputPortList = context.getInputPorts();
    if (inputPortList != null && !inputPortList.isEmpty()) {
      DefaultPartition.assignPartitionKeys(newPartitions, inputPortList.iterator().next());
    }
  } else {
    // define partitions is being called again
    if (context.getParallelPartitionCount() != 0) {
      newPartitions = repartitionParallel(partitions, context);
    } else if (partition.getPartitionKeys().isEmpty()) {
      newPartitions = repartitionInputOperator(partitions);
    } else {
      newPartitions = repartition(partitions);
    }
  }

  logger.debug("new partition size {}", newPartitions.size());
  return newPartitions;
}
 

private InputPort<Object> consoleOutput(DAG dag, String operatorName)
{
  if (PubSubHelper.isGatewayConfigured(dag)) {
    String topic = "examples.twitter." + operatorName;
    //LOG.info("WebSocket with gateway at: {}", gatewayAddress);
    PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
    wsOut.setUri(PubSubHelper.getURI(dag));
    wsOut.setTopic(topic);
    return wsOut.input;
  }
  ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
  operator.setStringFormat(operatorName + ": %s");
  return operator.input;
}
 

private InputPort<Object> wsOutput(DAG dag, String operatorName)
{
  if (PubSubHelper.isGatewayConfigured(dag)) {
    String appId = "appid";
    //appId = dag.attrValue(DAG.APPLICATION_ID, null); // will be used once UI is able to pick applications from list and listen to corresponding application
    String topic = "apps.logstream." + appId + "." + operatorName;
    PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
    wsOut.setUri(PubSubHelper.getURI(dag));
    wsOut.setTopic(topic);
    return wsOut.input;
  }
  ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
  operator.setStringFormat(operatorName + ": %s");
  return operator.input;
}
 

public InputPort<Map<String, Map<String, Number>>> getRedisOutput(String name, DAG dag, int dbIndex)
{
  @SuppressWarnings("unchecked")
  RedisNumberSummationMapOutputOperator<String, Map<String, Number>> oper = dag.addOperator(name, RedisNumberSummationMapOutputOperator.class);
  oper.getStore().setDbIndex(dbIndex);
  return oper.input;
}
 

@Override
public void addInputPort(InputPort<?> portObject, Field field, InputPortFieldAnnotation portAnnotation, AppData.QueryPort adqAnnotation)
{
  if (!OperatorMeta.this.inputStreams.isEmpty()) {
    for (Map.Entry<LogicalPlan.InputPortMeta, LogicalPlan.StreamMeta> e : OperatorMeta.this.inputStreams.entrySet()) {
      LogicalPlan.InputPortMeta pm = e.getKey();
      if (pm.operatorMeta == OperatorMeta.this && pm.fieldName.equals(field.getName())) {
        //LOG.debug("Found existing port meta for: " + field);
        inPortMap.put(portObject, pm);
        markInputPortIfHidden(pm.getPortName(), pm, field.getDeclaringClass());
        return;
      }
    }
  }
  InputPortMeta metaPort = new InputPortMeta();
  metaPort.operatorMeta = OperatorMeta.this;
  metaPort.fieldName = field.getName();
  metaPort.portAnnotation = portAnnotation;
  metaPort.adqAnnotation = adqAnnotation;
  inPortMap.put(portObject, metaPort);
  markInputPortIfHidden(metaPort.getPortName(), metaPort, field.getDeclaringClass());
  if (metaPort.getStreamCodec() == null) {
    metaPort.setStreamCodec(portObject.getStreamCodec());
  } else if (portObject.getStreamCodec() != null) {
    LOG.info("Operator {} input port {} attribute {} overrides codec {} with {} codec", metaPort.getOperatorMeta().getName(),
        metaPort.getPortName(), STREAM_CODEC.getSimpleName(), portObject.getStreamCodec(), metaPort.getStreamCodec());
  }
}
 

/**
 * Assign partitions keys for the given list of partitions and port of the logical operator.
 * <p>
 * The incoming stream will be partitioned by n keys, with n the nearest power of 2 greater or equal to the
 * number of partition instances provided. If the number of instances does not align with a power of 2, some of the
 * partitions will be assigned 2 keys. This logic is used for default partitioning and can be used to implement
 * {@link Partitioner}.
 *
 * @param <T>        Type of the partitionable object
 * @param partitions
 * @param inputPort
 */
public static <T> void assignPartitionKeys(Collection<Partition<T>> partitions, InputPort<?> inputPort)
{
  if (partitions.isEmpty()) {
    throw new IllegalArgumentException("partitions collection cannot be empty");
  }

  int partitionBits = (Integer.numberOfLeadingZeros(0) - Integer.numberOfLeadingZeros(partitions.size() - 1));
  int partitionMask = 0;
  if (partitionBits > 0) {
    partitionMask = -1 >>> (Integer.numberOfLeadingZeros(-1)) - partitionBits;
  }

  Iterator<Partition<T>> iterator = partitions.iterator();
  for (int i = 0; i <= partitionMask; i++) {
    Partition<?> p;
    if (iterator.hasNext()) {
      p = iterator.next();
    } else {
      iterator = partitions.iterator();
      p = iterator.next();
    }

    PartitionKeys pks = p.getPartitionKeys().get(inputPort);
    if (pks == null) {
      p.getPartitionKeys().put(inputPort, new PartitionKeys(partitionMask, Sets.newHashSet(i)));
    } else {
      pks.partitions.add(i);
    }
  }
}
 

private boolean isInputPortConnectedToDelayOperator(String portName)
{
  Operators.PortContextPair<InputPort<?>> pcPair = descriptor.inputPorts.get(portName);
  if (pcPair == null || pcPair.context == null) {
    return false;
  }
  return pcPair.context.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR);
}
 

/**
 * Populate {@link #reservoirPortMap} with information on which reservoirs are connected to which input ports
 */
protected void populateReservoirInputPortMap()
{
  for (Entry<String,Operators.PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
    if (entry.getValue().component != null && entry.getValue().component instanceof InputPort) {
      if (inputs.containsKey(entry.getKey())) {
        reservoirPortMap.put(inputs.get(entry.getKey()), entry.getValue().component.getSink());
      }
    }
  }
}
 

@Override
public void putAll(Map<? extends InputPort<?>, ? extends PartitionKeys> m)
{
  for (Map.Entry<? extends InputPort<?>, ? extends PartitionKeys> entry: m.entrySet()) {
    put(entry.getKey(), entry.getValue());
  }
}
 

@Override
public InputPort<?> getPort()
{
  for (Map.Entry<InputPort<?>, InputPortMeta> e : operatorMeta.getPortMapping().inPortMap.entrySet()) {
    if (e.getValue() == this) {
      return e.getKey();
    }
  }
  throw new AssertionError("Cannot find the port object for " + this);
}