com.datatorrent.api.DefaultOutputPort Java Examples

/**
 * This is a utility method which is used to attach the output port of an {@link EmbeddableQueryInfoProvider} to the input port
 * of the encapsulating {@link AppData.Store}.
 * @param <T> The type of data emitted by the {@link EmbeddableQueryInfoProvider}'s output port and received by the
 * {@link AppData.Store}'s input port.
 * @param outputPort The output port of the {@link EmbeddableQueryInfoProvider} which is being used by an {@link AppData.Store}.
 * @param inputPort The input port of the {@link AppData.Store} which is using an {@link EmbeddableQueryInfoProvider}.
 */
public static <T> void attachOutputPortToInputPort(DefaultOutputPort<T> outputPort, final DefaultInputPort<T> inputPort)
{
  outputPort.setSink(new Sink<Object>()
  {
    @Override
    @SuppressWarnings("unchecked")
    public void put(Object tuple)
    {
      LOG.debug("processing tuple");
      inputPort.process((T)tuple);
    }

    @Override
    public int getCount(boolean reset)
    {
      return 0;
    }

  });
}
 

@SuppressWarnings("unchecked")
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  KeyGen keyGen = dag.addOperator("KeyGenerator", new KeyGen());
  UniqueValueCount<Integer> valCount = dag.addOperator("ValueCounter", new UniqueValueCount<Integer>());
  IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("Unique", new IntegerUniqueValueCountAppender());
  VerifyTable verifyTable = dag.addOperator("VerifyTable", new VerifyTable());

  @SuppressWarnings("rawtypes")
  DefaultOutputPort valOut = valCount.output;
  @SuppressWarnings("rawtypes")
  DefaultOutputPort uniqueOut = uniqueUnifier.output;
  dag.addStream("DataIn", keyGen.output, valCount.input);
  dag.addStream("UnifyWindows", valOut, uniqueUnifier.input);
  dag.addStream("ResultsOut", uniqueOut, verifyTable.input);
}
 

@SuppressWarnings("unchecked")
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
  UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
  ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
  IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("StatefulUniqueCounter", new IntegerUniqueValueCountAppender());
  dag.getOperatorMeta("StatefulUniqueCounter").getMeta(uniqueUnifier.input).getAttributes().put(Context.PortContext.STREAM_CODEC, new KeyBasedStreamCodec());

  @SuppressWarnings("rawtypes")
  DefaultOutputPort valOut = valCount.output;
  @SuppressWarnings("rawtypes")
  DefaultOutputPort uniqueOut = uniqueUnifier.output;

  dag.addStream("Events", randGen.outport, valCount.input);
  dag.addStream("Unified", valOut, uniqueUnifier.input);
  dag.addStream("Result", uniqueOut, consOut.input);
}
 

@Override
protected void emitUpdates()
{
  super.emitUpdates();

  for (int index = 0; index < aggregatorsInfo.size(); ++index) {
    MutablePair<Integer, Integer> info = aggregatorsInfo.get(index);
    if (info == null) {
      continue;
    }

    int aggregatorID = info.left;
    int dimensionDescriptorID = info.right;
    DefaultOutputPort<List<Aggregate>> outputPort = getOutputPort(index++, aggregatorID, dimensionDescriptorID);
    if (outputPort == null || !outputPort.isConnected()) {
      continue;
    }

    updatingAggregates.clear();

    for (Map.Entry<EventKey, Aggregate> entry : cache.entrySet()) {
      if (aggregatorID == entry.getKey().getAggregatorID()
          && entry.getKey().getDimensionDescriptorID() == dimensionDescriptorID
          && getMaxTimestamp() == entry.getKey().getKey().getFieldLong(TIME_FIELD_NAME)) {
        updatingAggregates.add(entry.getValue());
      }
    }

    if (!updatingAggregates.isEmpty()) {
      outputPort.emit(updatingAggregates);
    }
  }
}
 

public QueryManagerAsynchronous(DefaultOutputPort<String> resultPort,
    QueueManager<QUERY_TYPE, META_QUERY, QUEUE_CONTEXT> queueManager,
    QueryExecutor<QUERY_TYPE, META_QUERY, QUEUE_CONTEXT, RESULT> queryExecutor,
    MessageSerializerFactory messageSerializerFactory,
    Thread mainThread)
{
  setResultPort(resultPort);
  setQueueManager(queueManager);
  setQueryExecutor(queryExecutor);
  setMessageSerializerFactory(messageSerializerFactory);
  setMainThread(mainThread);
}
 

private void emitCreditCardKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, CreditCardData>> outputPort)
{
  MerchantKey key = getMerchantKey(tuple);

  CreditCardData data = new CreditCardData();
  data.fullCcNum = tuple.fullCcNum;
  data.amount = tuple.amount;

  KeyValPair<MerchantKey, CreditCardData> keyValPair = new KeyValPair<MerchantKey, CreditCardData>(key, data);
  outputPort.emit(keyValPair);
}
 

public DefaultOutputPort<DimensionTuple> populateUpstreamDAG(DAG dag, Configuration configuration)
{
  JsonGenerator eventGenerator = dag.addOperator("eventGenerator", new JsonGenerator());
  FilterTuples filterTuples = dag.addOperator("filterTuples", new FilterTuples());
  FilterFields filterFields = dag.addOperator("filterFields", new FilterFields());
  
  // Connect the Ports in the Operators
  dag.addStream("filterTuples", eventGenerator.out, filterTuples.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("filterFields", filterTuples.output, filterFields.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
  
  TupleToDimensionTupleConverter converter = dag.addOperator("converter", new TupleToDimensionTupleConverter());
  
  if(includeRedisJoin) {
    RedisJoin redisJoin = dag.addOperator("redisJoin", new RedisJoin());
    dag.addStream("redisJoin", filterFields.output, redisJoin.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
    dag.addStream("converter", redisJoin.output, converter.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);

    dag.setInputPortAttribute(redisJoin.input, Context.PortContext.PARTITION_PARALLEL, true);

    setupRedis(eventGenerator.getCampaigns());
  } else {
    dag.addStream("convert", filterFields.output, converter.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
  }
  

  dag.setInputPortAttribute(filterTuples.input, Context.PortContext.PARTITION_PARALLEL, true);
  dag.setInputPortAttribute(filterFields.input, Context.PortContext.PARTITION_PARALLEL, true);
  dag.setInputPortAttribute(converter.inputPort, Context.PortContext.PARTITION_PARALLEL, true);

  dag.setAttribute(eventGenerator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<EventGenerator>(PARTITION_NUM));

  return converter.outputPort;
}
 

@Override
public void populateDAG(DAG dag, Configuration configuration)
{
  redisServer = configuration.get("dt.application.AppWithDCWithoutDe.redisServer");
  
  DefaultOutputPort<DimensionTuple> upstreamOutput = populateUpstreamDAG(dag, configuration);

  //populateHardCodedDimensionsDAG(dag, configuration, generateOperator.outputPort);
  populateDimensionsDAG(dag, configuration, upstreamOutput);
}
 

public void populateDimensionsDAG(DAG dag, Configuration conf, DefaultOutputPort<DimensionTuple> upstreamPort) 
{
  final String eventSchema = SchemaUtils.jarResourceFileToString(eventSchemaLocation);
  
  // dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);

  // Set operator properties
  // key expression
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("campaignId", DimensionTuple.CAMPAIGNID);
    keyToExpression.put("time", DimensionTuple.EVENTTIME);
    dimensions.setKeyToExpression(keyToExpression);
  }

  // aggregate expression
  {
    Map<String, String> valueToExpression = Maps.newHashMap();
    valueToExpression.put("clicks", DimensionTuple.CLICKS);
    valueToExpression.put("latency", DimensionTuple.LATENCY);
    
    dimensions.setAggregateToExpression(valueToExpression);
  }

  // event schema
  dimensions.setConfigurationSchemaJSON(eventSchema);
  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());

  dag.setUnifierAttribute(dimensions.output, OperatorContext.MEMORY_MB, 10240);
  
  dag.setInputPortAttribute(dimensions.input, Context.PortContext.PARTITION_PARALLEL, true);
  
  // store
  AppDataSingleSchemaDimensionStoreHDHT store = createStore(dag, conf, eventSchema); 
  store.setCacheWindowDuration(10000 * 5 / STREAMING_WINDOW_SIZE_MILLIS);   //cache for 5 windows
  dag.addStream("GenerateStream", upstreamPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  
  StoreStreamCodec codec = new StoreStreamCodec();
  dag.setInputPortAttribute(store.input, PortContext.STREAM_CODEC, codec);
  dag.addStream("DimensionalStream", dimensions.output, store.input);

  
  if (includeQuery) {
    createQuery(dag, conf, store);

    // wsOut
    PubSubWebSocketAppDataResult wsOut = createQueryResult(dag, conf, store);

    dag.addStream("QueryResult", store.queryResult, wsOut.input);
  } else {
    DevNull devNull = new DevNull();
    dag.addOperator("devNull", devNull);
    dag.addStream("QueryResult", store.queryResult, devNull.data);
  }
  
  dag.setAttribute(DAGContext.STREAMING_WINDOW_SIZE_MILLIS, STREAMING_WINDOW_SIZE_MILLIS);
}
 

private void emitMerchantKeyTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<MerchantKey, Long>> outputPort)
{
  MerchantKey key = getMerchantKey(tuple);
  KeyValPair<MerchantKey, Long> keyValPair = new KeyValPair<MerchantKey, Long>(key, tuple.amount);
  outputPort.emit(keyValPair);
}
 

private void emitBankIdNumTuple(MerchantTransaction tuple, DefaultOutputPort<KeyValPair<KeyValPair<MerchantKey, String>, Integer>> outputPort)
{
  MerchantKey key = getMerchantKey(tuple);
  KeyValPair<MerchantKey, String> keyValPair = new KeyValPair<MerchantKey, String>(key, tuple.bankIdNum);
  outputPort.emit(new KeyValPair<KeyValPair<MerchantKey, String>, Integer>(keyValPair, 1));
}
 

@Override
protected DefaultOutputPort<List<Aggregate>> getOutputPort(int index, int aggregatorID, int dimensionDescriptorID)
{
  return serviceCallOutputPort;
}
 

protected void emitUpdatesAverageFor(String fieldName, int dimensionDesciptionId,
    DefaultOutputPort<List<Map<String, Long>>> output)
{
  if (!output.isConnected()) {
    return;
  }

  long sum = 0;
  long count = 0;
  for (Map.Entry<EventKey, Aggregate> entry : cache.entrySet()) {
    //      logger.debug("\nentry.getKey().getDimensionDescriptorID(): {},  dimensionDesciptionId: {}; \n"
    //          + "entry.getKey().getKey().getFieldInt(DimensionsDescriptor.DIMENSION_TIME_BUCKET): {}, timeBucket: {}; \n"
    //          + "entry.getKey().getKey().getFieldLong(TIME_FIELD_NAME): {}, getMaxTimestamp(): {}", 
    //          entry.getKey().getDimensionDescriptorID(), dimensionDesciptionId, 
    //          entry.getKey().getKey().getFieldInt(DimensionsDescriptor.DIMENSION_TIME_BUCKET), timeBucket, 
    //          entry.getKey().getKey().getFieldLong(TIME_FIELD_NAME), getMaxTimestamp()); 
    //check the dimension id;
    if (entry.getKey().getDimensionDescriptorID() != dimensionDesciptionId) {
      continue;
    }
    //time bucket is 1 minute
    if (entry.getKey().getKey().getFieldInt(DimensionsDescriptor.DIMENSION_TIME_BUCKET) != timeBucket) {
      continue;
    }
    //the time is most recent time
    long time = entry.getKey().getKey().getFieldLong(TIME_FIELD_NAME);
    if (time != getMaxTimestamp()) {
      continue;
    }

    //this is the qualified entry 
    int aggregatorId = entry.getKey().getAggregatorID();
    if (AggregatorIncrementalType.COUNT.ordinal() == aggregatorId) {
      count = entry.getValue().getAggregates().getFieldLong(fieldName);
    }
    if (AggregatorIncrementalType.SUM.ordinal() == aggregatorId) {
      sum = entry.getValue().getAggregates().getFieldLong(fieldName);
    }

    if (sum != 0 && count != 0) {
      break;
    }
  }
  if (count != 0) {
    fieldValue.clear();
    //fieldValue.put(fieldName, sum/count);
    fieldValue.put("current", sum / count); //change the field name to current
    output.emit(averageTuple);
  }
  logger.info("field name: {}; sum={}; count={}", fieldName, sum, count);
}
 

@Override
protected DefaultOutputPort getOutputPortOfGenerator( StringGeneratorInputOperator generator )
{
  return generator.outputPort;
}
 

@Override
protected DefaultOutputPort getOutputPortOfGenerator(POJOTupleGenerateOperator generator)
{
  return generator.outputPort;
}
 

protected abstract DefaultOutputPort<List<Aggregate>> getOutputPort(int index, int aggregatorID,
int dimensionDescriptorID);
 

private void setResultPort(DefaultOutputPort<String> resultPort)
{
  this.resultPort = Preconditions.checkNotNull(resultPort);
}
 

@Override
protected DefaultOutputPort<List<Aggregate>> getOutputPort(int index, int aggregatorID, int dimensionDescriptorID)
{
  return updateWithList;
}
 

@Override
public DefaultOutputPort<String> getOutputPort()
{
  return outputPort;
}
 

@Test
public void stressTest() throws Exception
{
  final int totalTuples = 100000;
  final int batchSize = 100;
  final double waitMillisProb = .01;

  AppDataWindowEndQueueManager<MockQuery, Void> queueManager = new AppDataWindowEndQueueManager<MockQuery, Void>();

  DefaultOutputPort<String> outputPort = new DefaultOutputPort<String>();
  CollectorTestSink<MockResult> sink = new CollectorTestSink<MockResult>();
  TestUtils.setSink(outputPort, sink);

  MessageSerializerFactory msf = new MessageSerializerFactory(new ResultFormatter());

  QueryManagerAsynchronous<MockQuery, Void, MutableLong, MockResult> queryManagerAsynch =
      new QueryManagerAsynchronous<>(outputPort, queueManager, new NOPQueryExecutor(waitMillisProb), msf,
      Thread.currentThread());

  Thread producerThread = new Thread(new ProducerThread(queueManager, totalTuples, batchSize, waitMillisProb));
  producerThread.start();
  producerThread.setName("Producer Thread");

  long startTime = System.currentTimeMillis();

  queryManagerAsynch.setup(null);

  int numWindows = 0;

  for (; sink.collectedTuples.size() < totalTuples && ((System.currentTimeMillis() - startTime) < 60000);
      numWindows++) {
    queryManagerAsynch.beginWindow(numWindows);
    Thread.sleep(100);
    queryManagerAsynch.endWindow();
  }

  producerThread.stop();
  queryManagerAsynch.teardown();

  try {
    Thread.sleep(1000);
  } catch (InterruptedException e) {
    //Do Nothing
  }

  Assert.assertEquals(totalTuples, sink.collectedTuples.size());
}
 

/**
 * Gets the output port for queries.
 * @return The output port for queries.
 */
DefaultOutputPort<QUERY_TYPE> getOutputPort();
 

protected abstract DefaultOutputPort getOutputPortOfGenerator( G generator );