com.datatorrent.api.Operator Java Examples

@GET
@Path(PATH_OPERATOR_CLASSES + "/{className}")
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings("unchecked")
public JSONObject describeOperator(@PathParam("className") String className)
{
  init();
  if (className == null) {
    throw new UnsupportedOperationException();
  }
  try {
    Class<?> clazz = Class.forName(className);
    if (Operator.class.isAssignableFrom(clazz)) {
      return operatorDiscoverer.describeOperator(className);
    } else {
      throw new NotFoundException();
    }
  } catch (Exception ex) {
    throw new NotFoundException();
  }
}
 

@Override
public <T extends Operator> T addOperator(@Nonnull String name, T operator)
{
  checkArgument(!isNullOrEmpty(name), "operator name is null or empty");

  if (operators.containsKey(name)) {
    if (operators.get(name).operator == operator) {
      return operator;
    }
    throw new IllegalArgumentException("duplicate operator name: " + operators.get(name));
  }

  // Avoid name conflict with module.
  if (modules.containsKey(name)) {
    throw new IllegalArgumentException("duplicate operator name: " + operators.get(name));
  }
  OperatorMeta decl = new OperatorMeta(name, operator);
  rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator
  leafOperators.add(decl); // will be removed when a sink is added to an output port for this operator
  operators.put(name, decl);
  return operator;
}
 

public List<CompactFieldNode> getAllOutputPorts(String clazzName)
{
  TypeGraphVertex tgv = typeGraph.get(clazzName);
  List<CompactFieldNode> ports = new ArrayList<>();
  TypeGraphVertex portVertex = typeGraph.get(Operator.OutputPort.class.getName());
  getAllPortsWithAncestor(portVertex, tgv, ports);
  Collections.sort(ports, new Comparator<CompactFieldNode>()
  {
    @Override
    public int compare(CompactFieldNode a, CompactFieldNode b)
    {
      return a.getName().compareTo(b.getName());
    }
  });
  return ports;
}
 

@Override
public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
{
  BeanMap beanMap = new BeanMap(operator);
  Map<String, Object> propertyValue = new HashMap<>();
  if (propertyName != null) {
    if (beanMap.containsKey(propertyName)) {
      propertyValue.put(propertyName, beanMap.get(propertyName));
    }
  } else {
    Iterator entryIterator = beanMap.entryIterator();
    while (entryIterator.hasNext()) {
      Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
      propertyValue.put(entry.getKey(), entry.getValue());
    }
  }
  logger.debug("Getting property {} on operator {}", propertyValue, operator);
  OperatorResponse response = new OperatorResponse(requestId, propertyValue);
  return response;
}
 

public void removeOperator(Operator operator)
{
  OperatorMeta om = getMeta(operator);
  if (om == null) {
    return;
  }

  Map<InputPortMeta, StreamMeta> inputStreams = om.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> e : inputStreams.entrySet()) {
    StreamMeta stream = e.getValue();
    if (e.getKey().getOperatorMeta() == om) {
      stream.sinks.remove(e.getKey());
    }
    // If persistStream was enabled for stream, reset stream when sink removed
    stream.resetStreamPersistanceOnSinkRemoval(e.getKey());
  }
  this.operators.remove(om.getName());
  rootOperators.remove(om);
  leafOperators.remove(om);
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

/**
 * Adjust the partitions of an input operator (operator with no connected input stream).
 *
 * @param <T> The operator type
 * @param partitions
 * @return The new operators.
 */
public static <T extends Operator> Collection<Partition<T>> repartitionInputOperator(Collection<Partition<T>> partitions)
{
  List<Partition<T>> newPartitions = new ArrayList<>();
  List<Partition<T>> lowLoadPartitions = new ArrayList<>();
  for (Partition<T> p: partitions) {
    int load = p.getLoad();
    if (load < 0) {
      if (!lowLoadPartitions.isEmpty()) {
        newPartitions.add(lowLoadPartitions.remove(0));
      } else {
        lowLoadPartitions.add(p);
      }
    } else if (load > 0) {
      newPartitions.add(new DefaultPartition<>(p.getPartitionedInstance()));
      newPartitions.add(new DefaultPartition<>(p.getPartitionedInstance()));
    } else {
      newPartitions.add(p);
    }
  }
  newPartitions.addAll(lowLoadPartitions);
  return newPartitions;
}
 

public Operator.Unifier<?> getUnifier()
{
  for (Map.Entry<OutputPort<?>, OutputPortMeta> e : operatorMeta.getPortMapping().outPortMap.entrySet()) {
    if (e.getValue() == this) {
      Unifier<?> unifier = e.getKey().getUnifier();
      if (unifier == null) {
        break;
      }
      LOG.debug("User supplied unifier is {}", unifier);
      return unifier;
    }
  }

  LOG.debug("Using default unifier for {}", this);
  return new DefaultUnifier();
}
 

@Override
public StreamMeta setSource(Operator.OutputPort<?> port)
{
  if (port instanceof ProxyOutputPort) {
    proxySource = port;
    return this;
  }
  OutputPortMeta portMeta = assertGetPortMeta(port);
  OperatorMeta om = portMeta.getOperatorMeta();
  if (om.outputStreams.containsKey(portMeta)) {
    String msg = String.format("Operator %s already connected to %s", om.name, om.outputStreams.get(portMeta).id);
    throw new IllegalArgumentException(msg);
  }
  this.source = portMeta;
  om.outputStreams.put(portMeta, this);
  return this;
}
 

private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint)
{
  try {
    LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo);
    long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId;
    StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
    agent.save(oo, oper.id, windowId);
    if (agent instanceof AsyncStorageAgent) {
      ((AsyncStorageAgent)agent).flush(oper.id, windowId);
    }
  } catch (IOException e) {
    // inconsistent state, no recovery option, requires shutdown
    throw new IllegalStateException("Failed to write operator state after partition change " + oper, e);
  }
  oper.setRecoveryCheckpoint(checkpoint);
  if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) {
    oper.checkpoints.add(checkpoint);
  }
}
 

@Override
@SuppressWarnings("unchecked")
public ApexStream<T> with(DAG.Locality locality)
{
  if (lastBrick.lastStream != null) {
    for (DagMeta.NodeMeta parent : lastBrick.nodeMeta.getParent()) {
      Pair<List<Operator.InputPort>, DAG.Locality> p = parent.getNodeStreams().get(lastBrick.lastStream.getLeft());
      if (p != null) {
        p.setValue(locality);
      }
    }
  }
  return this;
}
 

@Override
public OperatorResponse execute(Operator oper, int arg1, long arg2) throws IOException
{
  if (oper instanceof TestOperator) {
    LOG.debug("Setting property");
    ((TestOperator)oper).propVal = true;
  }
  return new TestOperatorResponse();
}
 

@Override
public RelInfo populateInputDAG(DAG dag, JavaTypeFactory typeFactory)
{
  CsvParser csvParser = dag.addOperator(OperatorUtils.getUniqueOperatorName("CSVParser"), CsvParser.class);
  csvParser.setSchema((String)operands.get(CSV_SCHEMA));

  return new RelInfo("CSVParser", Lists.<Operator.InputPort>newArrayList(csvParser.in), csvParser, csvParser.out,
    getRowType(typeFactory));
}
 

@SuppressWarnings("deprecation")
@Override
public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
{
  cmd.execute(operator,operatorId,windowId);
  return null;
}
 

private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
 

public Operator loadOperator(PTOperator oper)
{
  try {
    LOG.debug("Loading state for {}", oper);
    return (Operator)oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).load(oper.id, oper.isOperatorStateLess() ? Stateless.WINDOW_ID : oper.recoveryCheckpoint.windowId);
  } catch (IOException e) {
    throw new RuntimeException("Failed to read partition state for " + oper, e);
  }
}
 

public void findInvalidDelays(OperatorMeta om, List<List<String>> invalidDelays, Stack<OperatorMeta> stack)
{
  stack.push(om);

  // depth first successors traversal
  boolean isDelayOperator = om.getOperator() instanceof Operator.DelayOperator;
  if (isDelayOperator) {
    if (om.getValue(OperatorContext.APPLICATION_WINDOW_COUNT) != 1) {
      LOG.debug("detected DelayOperator having APPLICATION_WINDOW_COUNT not equal to 1");
      invalidDelays.add(Collections.singletonList(om.getName()));
    }
  }

  for (StreamMeta downStream: om.outputStreams.values()) {
    for (InputPortMeta sink : downStream.sinks) {
      OperatorMeta successor = sink.getOperatorMeta();
      if (isDelayOperator) {
        sink.attributes.put(IS_CONNECTED_TO_DELAY_OPERATOR, true);
        // Check whether all downstream operators are already visited in the path
        if (successor != null && !stack.contains(successor)) {
          LOG.debug("detected DelayOperator does not immediately output to a visited operator {}.{}->{}.{}",
              om.getName(), downStream.getSource().getPortName(), successor.getName(), sink.getPortName());
          invalidDelays.add(Arrays.asList(om.getName(), successor.getName()));
        }
      } else {
        findInvalidDelays(successor, invalidDelays, stack);
      }
    }
  }
  stack.pop();
}
 

@SuppressWarnings("unchecked")
public Class<? extends Operator> getOperatorClass(String className) throws ClassNotFoundException
{
  if (CollectionUtils.isEmpty(operatorClassNames)) {
    loadGenericOperatorClasses();
  }

  Class<?> clazz = classLoader.loadClass(className);

  if (!Operator.class.isAssignableFrom(clazz)) {
    throw new IllegalArgumentException("Argument must be a subclass of Operator class");
  }
  return (Class<? extends Operator>)clazz;
}
 

@Override
public void addOutputPort(Operator.OutputPort<?> port, Field field, OutputPortFieldAnnotation portAnnotation, AppData.ResultPort adrAnnotation)
{
  if (!outputPorts.containsKey(field.getName())) {
    outputPorts.put(field.getName(), new PortContextPair<OutputPort<?>>(port));
  }
}
 

@Override
public void execute(PlanModifier pm)
{
  Class<? extends Operator> operClass = StramUtils.classForName(operatorFQCN, Operator.class);
  Operator operator = StramUtils.newInstance(operClass);
  pm.addOperator(operatorName, operator);
}
 

public NodeMeta addNode(Operator operator, NodeMeta parent, Operator.OutputPort parentOutput, Operator.InputPort inputPort, Option... options)
{

  NodeMeta newNode = new NodeMeta(operator, options);
  if (parent == null) {
    heads.add(newNode);
  } else {
    parent.nodeStreams.get(parentOutput).getLeft().add(inputPort);
    parent.children.add(newNode);
    newNode.parent.add(parent);
  }
  return newNode;
}
 

RepartitionContext(Partitioner<Operator> partitioner, PMapping currentMapping, int partitionCount)
{
  super(currentMapping, partitionCount);
  this.operators = currentMapping.partitions;
  this.currentPartitions = new ArrayList<>(operators.size());
  this.currentPartitionMap = Maps.newHashMapWithExpectedSize(operators.size());
  this.operatorIdToPartition = Maps.newHashMapWithExpectedSize(operators.size());

  // collect current partitions with committed operator state
  // those will be needed by the partitioner for split/merge
  for (PTOperator pOperator : operators) {
    Map<InputPort<?>, PartitionKeys> pks = pOperator.getPartitionKeys();
    if (pks == null) {
      throw new AssertionError("Null partition: " + pOperator);
    }

    // if partitions checkpoint at different windows, processing for new or modified
    // partitions will start from earliest checkpoint found (at least once semantics)
    if (minCheckpoint == null) {
      minCheckpoint = pOperator.recoveryCheckpoint;
    } else if (minCheckpoint.windowId > pOperator.recoveryCheckpoint.windowId) {
      minCheckpoint = pOperator.recoveryCheckpoint;
    }

    Operator partitionedOperator = loadOperator(pOperator);
    DefaultPartition<Operator> partition = new DefaultPartition<>(partitionedOperator, pks, pOperator.loadIndicator, pOperator.stats);
    currentPartitions.add(partition);
    currentPartitionMap.put(partition, pOperator);
    LOG.debug("partition load: {} {} {}", pOperator, partition.getPartitionKeys(), partition.getLoad());
    operatorIdToPartition.put(pOperator.getId(), partition);
  }

  newPartitions = partitioner.definePartitions(new ArrayList<Partition<Operator>>(currentPartitions), this);
}
 

@Override
public void handle(DAGSetupEvent event)
{
  for (DAG.OperatorMeta ometa : dag.getAllOperatorsMeta()) {
    Operator o = ometa.getOperator();
    LogicalPlanConfiguration.setOperatorProperties(o, propertyMap);
  }
}
 

@Override
public OperatorMeta getMeta(Operator operator)
{
  // TODO: cache mapping
  for (OperatorMeta o: getAllOperators()) {
    if (o.operator == operator) {
      return o;
    }
  }
  throw new IllegalArgumentException("Operator not associated with the DAG: " + operator);
}
 

@Override
@SuppressWarnings("unchecked")
public ApexStreamImpl<T> print()
{
  ConsoleOutputOperator consoleOutputOperator = new ConsoleOutputOperator();
  addOperator(consoleOutputOperator,
      (Operator.InputPort<T>)consoleOutputOperator.input, null, Option.Options.name(IDGenerator.generateOperatorIDWithUUID(consoleOutputOperator.getClass())));
  return this;
}
 

@Test
public void testAtMostOnceProcessingModeValidation()
{
  TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
  TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);

  GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
  dag.setOperatorAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);

  dag.addStream("input1.outport", input1.outport, amoOper.inport1);
  dag.addStream("input2.outport", input2.outport, amoOper.inport2);

  GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
  dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
  dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);

  try {
    dag.validate();
    Assert.fail("Exception expected for " + outputOper);
  } catch (ValidationException ve) {
    Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/AT_MOST_ONCE");
  }
  dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, null);
  dag.validate();

  OperatorMeta outputOperOm = dag.getMeta(outputOper);
  Assert.assertEquals("" + outputOperOm.getAttributes(), Operator.ProcessingMode.AT_MOST_ONCE, outputOperOm.getValue(OperatorContext.PROCESSING_MODE));
}
 

public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias)
{
  if (PubSubHelper.isGatewayConfigured(dag)) {
    URI uri = PubSubHelper.getURI(dag);

    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());

    Map<String, String> conversionMap = Maps.newHashMap();
    conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);

    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    snapshotServer.setTableFieldToMapField(conversionMap);

    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.setUri(uri);
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);

    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setUri(uri);
    Operator.InputPort<String> queryResultPort = wsResult.input;

    dag.addStream("MapProvider", topCount, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
  } else {
    ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
    operator.setStringFormat(operatorName + ": %s");

    dag.addStream("MapProvider", topCount, operator.input);
  }
}
 

@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[]{sink1};
  return addStream(id, source, ports);
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
private void addDAGToCurrentDAG(ModuleMeta moduleMeta)
{
  LogicalPlan subDag = moduleMeta.getDag();
  String subDAGName = moduleMeta.getName();
  String name;
  for (OperatorMeta operatorMeta : subDag.getAllOperators()) {
    name = subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
    Operator op = this.addOperator(name, operatorMeta.getOperator());
    OperatorMeta operatorMetaNew = this.getMeta(op);
    operatorMetaNew.copyAttributesFrom(operatorMeta);
    operatorMetaNew.setModuleName(operatorMeta.getModuleName() == null ? subDAGName :
        subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getModuleName());
  }

  for (StreamMeta streamMeta : subDag.getAllStreams()) {
    OutputPortMeta sourceMeta = streamMeta.getSource();
    List<InputPort<?>> ports = new LinkedList<>();
    for (InputPortMeta inputPortMeta : streamMeta.getSinks()) {
      ports.add(inputPortMeta.getPort());
    }
    InputPort[] inputPorts = ports.toArray(new InputPort[]{});

    name = subDAGName + MODULE_NAMESPACE_SEPARATOR + streamMeta.getName();
    StreamMeta streamMetaNew = this.addStream(name, sourceMeta.getPort(), inputPorts);
    streamMetaNew.setLocality(streamMeta.getLocality());
  }
}
 

@Override
public <T extends Operator> T addOperator(@Nonnull String name, Class<T> clazz)
{
  T instance;
  try {
    instance = clazz.newInstance();
  } catch (Exception ex) {
    throw new IllegalArgumentException(ex);
  }
  addOperator(name, instance);
  return instance;
}