com.datatorrent.api.Context Java Examples

The following examples show how to use com.datatorrent.api.Context. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetFilePOJOReaderTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testParquetEmptyFile() throws Exception
{
  FileContext.getLocalFSFileContext().delete(new Path(new File(testMeta.dir).getAbsolutePath()), true);
  List<EventRecord> data = Lists.newArrayList();
  writeParquetFile(PARQUET_SCHEMA, new File(testMeta.dir, "data.parquet"), data);

  parquetFilePOJOReader.output.setSink(outputSink);
  parquetFilePOJOReader.setDirectory(testMeta.dir);
  parquetFilePOJOReader.setParquetSchema(PARQUET_SCHEMA);
  parquetFilePOJOReader.setup(testMeta.context);
  testMeta.portContext.getAttributes().put(Context.PortContext.TUPLE_CLASS, EventRecordV2.class);
  parquetFilePOJOReader.output.setup(testMeta.portContext);

  for (long wid = 0; wid < 2; wid++) {
    parquetFilePOJOReader.beginWindow(0);
    parquetFilePOJOReader.emitTuples();
    parquetFilePOJOReader.endWindow();
  }

  Assert.assertEquals("number tuples", 0, outputSink.collectedTuples.size());
  parquetFilePOJOReader.teardown();

}
 
Example #2
Source File: LogicalPlanConfiguration.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * This creates a {@link ConfElement}.
 *
 * @param element The current {@link StramElement} representing a {@link ConfElement}.
 * @param parent The parent {@link ConfElement}.
 * @param additionalRelatedElements Any additional {@link StramElement} that could be
 * related to this {@link ConfElement}.
 * @param contextClass The {@link Context} class that contains all the attributes to
 * be used by this {@link ConfElement}.
 */
ConfElement(StramElement element,
    ConfElement parent,
    Set<StramElement> additionalRelatedElements,
    Class<? extends Context> contextClass)
{
  this.element = element;
  this.parent = parent;

  if (additionalRelatedElements != null) {
    this.allRelatedElements.addAll(additionalRelatedElements);
  }

  this.allRelatedElements.add(element);

  this.contextClass = contextClass;

  this.contextAttributes = contextClass != null ? ContextUtils.CONTEXT_CLASS_TO_ATTRIBUTES.get(contextClass) : new HashSet<String>();
}
 
Example #3
Source File: AutoMetricTest.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Test
@Ignore
public void testMetricsAggregations() throws Exception
{
  CountDownLatch latch = new CountDownLatch(2);

  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
  MockAggregator aggregator = new MockAggregator(latch);
  dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  dag.setOperatorAttribute(o1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(2));

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");
  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  latch.await();
  Assert.assertEquals("progress", 2L, ((Long)aggregator.result.get("progress")).longValue());
  lc.shutdown();
}
 
Example #4
Source File: FixedWidthParser.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * Activate the Parser
 */
@Override
public void activate(Context context)
{
  try {
    if (clazz != null) {
      setters = new ArrayList<>();
      List<String> fieldNames = schema.getFieldNames();
      if (fieldNames != null) {
        for (String fieldName : fieldNames) {
          addSetter(fieldName);
        }
      }
    }
  } catch (Exception e) {
    logger.error("Cannot activate Parser Reason {}", e.getMessage());
    throw e;
  }
}
 
Example #5
Source File: EmbeddedAppLauncherImpl.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
private void addLibraryJarsToClasspath(LogicalPlan lp) throws MalformedURLException
{
  String libJarsCsv = lp.getAttributes().get(Context.DAGContext.LIBRARY_JARS);

  if (libJarsCsv != null && libJarsCsv.length() != 0) {
    String[] split = libJarsCsv.split(StramClient.LIB_JARS_SEP);
    if (split.length != 0) {
      URL[] urlList = new URL[split.length];
      for (int i = 0; i < split.length; i++) {
        File file = new File(split[i]);
        urlList[i] = file.toURI().toURL();
      }

      // Set class loader.
      ClassLoader prevCl = Thread.currentThread().getContextClassLoader();
      URLClassLoader cl = URLClassLoader.newInstance(urlList, prevCl);
      Thread.currentThread().setContextClassLoader(cl);
    }
  }

}
 
Example #6
Source File: TransformOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
protected void starting(Description description)
{
  super.starting(description);
  operator = new TransformOperator();

  sink = new CollectorTestSink<>();
  TestUtils.setSink(operator.output, sink);

  operator.setup(null);

  Attribute.AttributeMap inMap = new Attribute.AttributeMap.DefaultAttributeMap();
  inMap.put(Context.PortContext.TUPLE_CLASS, InputClass.class);
  operator.input.setup(new PortContext(inMap, null));

  Attribute.AttributeMap outMap = new Attribute.AttributeMap.DefaultAttributeMap();
  outMap.put(Context.PortContext.TUPLE_CLASS, OutputClass.class);
  operator.output.setup(new PortContext(outMap, null));
}
 
Example #7
Source File: AvroToPojo.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public void setup(PortContext context)
{
  setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS));

  columnFieldSetters = Lists.newArrayList();

  /**
   * Check if the mapping of Generic record fields to POJO is given, else
   * use reflection
   */
  if (getGenericRecordToPOJOFieldsMapping() == null) {
    setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass())));
  } else {
    setFieldInfos(createFieldInfoMap(getGenericRecordToPOJOFieldsMapping()));
  }

  initColumnFieldSetters(getFieldInfos());
  initializeActiveFieldSetters();
}
 
Example #8
Source File: HDHTBenchmarkApplication.java    From examples with Apache License 2.0 6 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  dag.setAttribute(DAG.APPLICATION_NAME, "HDHTBenchmarkApplication");
  Generator gen = dag.addOperator("Generator", new Generator());
  gen.setTupleBlast(1000);
  gen.setSleepms(0);
  dag.getOperatorMeta("Generator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 1);

  HDSOperator hdsOut = dag.addOperator("Store", new HDSOperator());
  TFileImpl.DTFileImpl hdsFile = new TFileImpl.DTFileImpl();
  hdsFile.setBasePath("WALBenchMarkDir");
  hdsOut.setFileStore(hdsFile);
  dag.getOperatorMeta("Store").getAttributes().put(Context.OperatorContext.COUNTERS_AGGREGATOR, new HDHTWriter.BucketIOStatAggregator());

  dag.addStream("s1", gen.out, hdsOut.input).setLocality(DAG.Locality.THREAD_LOCAL);
}
 
Example #9
Source File: LatencyTest.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Before
public void setup()
{
  dag = StramTestSupport.createDAG(testMeta);
  dag.setAttribute(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS, windowWidthMillis);
  dag.setAttribute(Context.DAGContext.HEARTBEAT_TIMEOUT_MILLIS, heartbeatTimeoutMillis);
  dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new StramTestSupport
      .MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);

  dag.addStream("o1.output1", o1.outport1, o3.inport1);
  dag.addStream("o2.output1", o2.outport1, o3.inport2);
  scm = new StreamingContainerManager(dag);
  PhysicalPlan plan = scm.getPhysicalPlan();
  o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
  o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
  o3p1 = plan.getOperators(dag.getMeta(o3)).get(0);
}
 
Example #10
Source File: EnrichmentOperator.java    From examples with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  try {
    loader = new FileLoader(filePath, backupResource);
    reloadData();
    lastScanTimeStamp = System.currentTimeMillis();

    if (updateKeys != null) {
      keyList = Lists.newArrayList(updateKeys.split(","));
    }

  } catch (IOException ex) {
    throw new RuntimeException("Failed to load mappings from the file.");
  }
}
 
Example #11
Source File: JsonSalesGenerator.java    From examples with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  AggregatorRegistry.DEFAULT_AGGREGATOR_REGISTRY.setup();

  schema = new DimensionalSchema(new DimensionalConfigurationSchema(eventSchemaJSON,
                                                            AggregatorRegistry.DEFAULT_AGGREGATOR_REGISTRY));

  maxProductId = 100;
  maxCustomerId = 30;
  maxChannelId = schema.getDimensionalConfigurationSchema().getKeysToEnumValuesList().get(KEY_CHANNEL).size();
  maxRegionId = schema.getDimensionalConfigurationSchema().getKeysToEnumValuesList().get(KEY_REGION).size();

  tuplesPerCurrentWindow = maxTuplesPerWindow;
  generateDiscounts();
  generateRegionalTax();
  initializeDataGenerators();
}
 
Example #12
Source File: LogicalPlanConfigurationTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testAttributesCodec()
{
  Assert.assertNotSame(null, new Long[] {com.datatorrent.api.Context.DAGContext.serialVersionUID, OperatorContext.serialVersionUID, PortContext.serialVersionUID});
  @SuppressWarnings("unchecked")
  Set<Class<? extends Context>> contextClasses = Sets.newHashSet(com.datatorrent.api.Context.DAGContext.class, OperatorContext.class, PortContext.class);
  for (Class<?> c : contextClasses) {
    for (Attribute<Object> attr : AttributeInitializer.getAttributes(c)) {
      Assert.assertNotNull(attr.name + " codec", attr.codec);
    }
  }
}
 
Example #13
Source File: SecurityUtilsTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
private void checkSecurityConfiguration(Configuration conf, boolean[][] securityConf)
{
  Assert.assertEquals("Number variations", 5, securityConf.length);
  SecurityUtils.init(conf);
  checkWebSecurity(securityConf[0][0], securityConf[0][1]);
  SecurityUtils.init(conf, Context.StramHTTPAuthentication.ENABLE);
  checkWebSecurity(securityConf[1][0], securityConf[1][1]);
  SecurityUtils.init(conf, Context.StramHTTPAuthentication.DISABLE);
  checkWebSecurity(securityConf[2][0], securityConf[2][1]);
  SecurityUtils.init(conf, Context.StramHTTPAuthentication.FOLLOW_HADOOP_AUTH);
  checkWebSecurity(securityConf[3][0], securityConf[3][1]);
  SecurityUtils.init(conf, Context.StramHTTPAuthentication.FOLLOW_HADOOP_HTTP_AUTH);
  checkWebSecurity(securityConf[4][0], securityConf[4][1]);
}
 
Example #14
Source File: StreamCodecTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testMxNMultipleStreamCodecs()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.setOperatorAttribute(node2, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  TestStreamCodec serDe = new TestStreamCodec();
  dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
  GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
  dag.setOperatorAttribute(node3, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  TestStreamCodec serDe2 = new TestStreamCodec();
  dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);

  dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
  LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);

  // Sanity check that physical operators have been allocated for n1meta and n2meta
  Assert.assertEquals("number operators " + n1meta.getName(), 2, plan.getOperators(n1meta).size());
  Assert.assertEquals("number operators " + n2meta.getName(), 3, plan.getOperators(n2meta).size());
  Assert.assertEquals("number operators " + n3meta.getName(), 3, plan.getOperators(n3meta).size());

  checkMxNStreamCodecs(node1, node2, node3, dnm);
}
 
Example #15
Source File: TwitterTopCounterApplication.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  // Setup the operator to get the data from twitter sample stream injected into the system.
  TwitterSampleInput twitterFeed = new TwitterSampleInput();
  twitterFeed = dag.addOperator("TweetSampler", twitterFeed);

  //  Setup the operator to get the URLs extracted from the twitter statuses
  TwitterStatusURLExtractor urlExtractor = dag.addOperator("URLExtractor", TwitterStatusURLExtractor.class);

  // Setup a node to count the unique urls within a window.
  UniqueCounter<String> uniqueCounter = dag.addOperator("UniqueURLCounter", new UniqueCounter<String>());
  // Get the aggregated url counts and count them over last 5 mins.
  dag.setAttribute(uniqueCounter, Context.OperatorContext.APPLICATION_WINDOW_COUNT, 600);
  dag.setAttribute(uniqueCounter, Context.OperatorContext.SLIDE_BY_WINDOW_COUNT, 1);


  WindowedTopCounter<String> topCounts = dag.addOperator("TopCounter", new WindowedTopCounter<String>());
  topCounts.setTopCount(10);
  topCounts.setSlidingWindowWidth(1);
  topCounts.setDagWindowWidth(1);

  // Feed the statuses from feed into the input of the url extractor.
  dag.addStream("TweetStream", twitterFeed.status, urlExtractor.input).setLocality(Locality.CONTAINER_LOCAL);
  //  Start counting the urls coming out of URL extractor
  dag.addStream("TwittedURLs", urlExtractor.url, uniqueCounter.data).setLocality(locality);
  // Count unique urls
  dag.addStream("UniqueURLCounts", uniqueCounter.count, topCounts.input);

  consoleOutput(dag, "topURLs", topCounts.output, SNAPSHOT_SCHEMA, "url");
}
 
Example #16
Source File: AbstractKafkaInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  applicationName = context.getValue(Context.DAGContext.APPLICATION_NAME);
  consumerWrapper.create(this);
  metrics = new KafkaMetrics(metricsRefreshInterval);
  windowDataManager.setup(context);
  operatorId = context.getId();
}
 
Example #17
Source File: StramLocalClusterTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  TestGeneratorInputOperator input = dag.addOperator("Input", new TestGeneratorInputOperator());
  test = dag.addOperator("Test", new DynamicLoader());

  dag.addStream("S1", input.outport, test.input);
  dag.setAttribute(Context.DAGContext.LIBRARY_JARS, generatedJar);
  dag.setInputPortAttribute(test.input, Context.PortContext.TUPLE_CLASS, pojo);
}
 
Example #18
Source File: KeyedWindowedOperatorImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  if (useUpdatedKeyStorage()) {
    updatedKeyStorage.getStore().setup(context);
    updatedKeyStorage.setup(context);
  }

  super.setup(context);
}
 
Example #19
Source File: TestPartition.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);

  long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
  id = context.getId();
  LOG.debug("Started setup, appWindowId = {}, operator id = {}", appWindowId, id);
}
 
Example #20
Source File: AbstractAppDataServer.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void activate(Context.OperatorContext ctx)
{
  if (embeddableQueryInfoProvider != null) {
    embeddableQueryInfoProvider.activate(ctx);
  }
}
 
Example #21
Source File: StramWebServicesTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testAttributes() throws Exception
{
  WebResource r = resource();
  ClientResponse response = r.path(StramWebServices.PATH + "/")
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  JSONObject attrs = json.getJSONObject("attributes");
  Assert.assertEquals(AutoMetricBuiltInTransport.class.getName() + ":xyz",
      attrs.getString(Context.DAGContext.METRICS_TRANSPORT.getSimpleName()));
}
 
Example #22
Source File: FilterTransformOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void activate(Context context)
{
  super.activate(context);
  if (condition != null) {
    conditionExpression = PojoUtils.createExpression(inputClass, condition, Boolean.class,
      expressionFunctions.toArray(new String[expressionFunctions.size()]));
  }
}
 
Example #23
Source File: StreamingContainer.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
private void deployNodes(List<OperatorDeployInfo> nodeList) throws IOException
{
  for (OperatorDeployInfo ndi : nodeList) {
    StorageAgent backupAgent = getValue(OperatorContext.STORAGE_AGENT, ndi);
    assert (backupAgent != null);

    Context parentContext;
    if (ndi instanceof UnifierDeployInfo) {
      OperatorContext unifiedOperatorContext = new OperatorContext(0, ndi.name,
          ((UnifierDeployInfo)ndi).operatorAttributes, containerContext);
      parentContext = new PortContext(ndi.inputs.get(0).contextAttributes, unifiedOperatorContext);
      massageUnifierDeployInfo(ndi);
    } else {
      parentContext = containerContext;
    }

    OperatorContext ctx = new OperatorContext(ndi.id, ndi.name, ndi.contextAttributes, parentContext);
    ctx.attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, ndi.checkpoint.windowId);
    Node<?> node = reuseOpNodes.get(ndi.id);
    if (node == null) {
      logger.info("Restoring operator {} to checkpoint {} stateless={}.", ndi.id, Codec.getStringWindowId(ndi.checkpoint.windowId), ctx.stateless);
      node = Node.retrieveNode(backupAgent.load(ndi.id, ctx.stateless ? Stateless.WINDOW_ID : ndi.checkpoint.windowId), ctx, ndi.type);
    } else {
      logger.info("Reusing previous operator instance {}", ndi.id);
      node = Node.retrieveNode(node.operator, ctx, ndi.type);
      node.setReuseOperator(true);
      reuseOpNodes.remove(ndi.id);
    }
    node.currentWindowId = ndi.checkpoint.windowId;
    node.applicationWindowCount = ndi.checkpoint.applicationWindowCount;
    node.firstWindowMillis = firstWindowMillis;
    node.windowWidthMillis = windowWidthMillis;

    node.setId(ndi.id);
    nodes.put(ndi.id, node);
    logger.debug("Marking operator {} as deployed.", node);
  }
}
 
Example #24
Source File: POJOInnerJoinOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  timeIncrement = context.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT) *
    context.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
  super.setup(context);
  for (int i = 0; i < 2; i++) {
    inputFieldObjects[i] = new FieldObjectMap();
  }
}
 
Example #25
Source File: KafkaOutputOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
private KafkaSinglePortExactlyOnceOutputOperator<Person> ResetKafkaOutput(
    String testName, Properties props, Context.OperatorContext operatorContext)
{
  KafkaSinglePortExactlyOnceOutputOperator<Person> kafkaOutput = new KafkaSinglePortExactlyOnceOutputOperator<>();
  kafkaOutput.setTopic(testName);
  kafkaOutput.setProperties(props);
  kafkaOutput.setup(operatorContext);

  return kafkaOutput;
}
 
Example #26
Source File: CustomControlTupleTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  Generator randomGenerator = dag.addOperator("randomGenerator", Generator.class);
  DefaultProcessor processor = dag.addOperator("process", DefaultProcessor.class);
  ControlAwareReceiver receiver = dag.addOperator("receiver", ControlAwareReceiver.class);
  dag.addStream("genToProcessor", randomGenerator.out, processor.input);
  dag.addStream("ProcessorToReceiver", processor.output, receiver.input);
  dag.setOperatorAttribute(processor, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
}
 
Example #27
Source File: KeyedWindowedMergeOperatorTestApplication.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  startingTime = System.currentTimeMillis();
  watermarkTime = System.currentTimeMillis() + 10000;
  i = 1;
}
 
Example #28
Source File: InnerJoinOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context.OperatorContext context)
{
  this.setExpiryTime(1L);
  // Number of buckets is set to 47000 because this is rounded number closer to sqrt of MAXINT. This guarantees
  // even distribution of keys across buckets.
  this.setNoOfBuckets(47000);
  this.setTimeFieldsStr("");
  super.setup(context);
}
 
Example #29
Source File: FSWindowDataManagerTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testDelete() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair1 = createManagerAndContextFor(1);
  pair1.second.getWal().setMaxLength(2);
  pair1.second.setup(pair1.first);

  Map<Integer, String> dataOf1 = Maps.newHashMap();
  dataOf1.put(1, "one");
  dataOf1.put(2, "two");
  dataOf1.put(3, "three");

  for (int i = 1; i <= 9; ++i) {
    pair1.second.save(dataOf1, i);
  }

  pair1.second.committed(3);
  pair1.second.teardown();

  Pair<Context.OperatorContext, FSWindowDataManager> pair1AfterRecovery = createManagerAndContextFor(1);
  testMeta.attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 1L);
  pair1AfterRecovery.second.setup(pair1AfterRecovery.first);

  Assert.assertEquals("window 1 deleted", null, pair1AfterRecovery.second.retrieve(1));
  Assert.assertEquals("window 3 deleted", null, pair1AfterRecovery.second.retrieve(3));

  Assert.assertEquals("window 4 exists", dataOf1, pair1AfterRecovery.second.retrieve(4));
  pair1.second.teardown();
}
 
Example #30
Source File: ApplicationWithGenerator.java    From streaming-benchmarks with Apache License 2.0 5 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration configuration)
{
   // Create operators for each step
   // settings are applied by the platform using the config file.
  JsonGenerator eventGenerator = dag.addOperator("eventGenerator", new JsonGenerator());
  FilterTuples filterTuples = dag.addOperator("filterTuples", new FilterTuples());
  FilterFields filterFields = dag.addOperator("filterFields", new FilterFields());
  RedisJoin redisJoin = dag.addOperator("redisJoin", new RedisJoin());
  CampaignProcessor campaignProcessor = dag.addOperator("campaignProcessor", new CampaignProcessor());

  eventGenerator.setNumAdsPerCampaign(Integer.parseInt(configuration.get("numberOfAds")));
  eventGenerator.setNumCampaigns(Integer.parseInt(configuration.get("numberOfCampaigns")));
  setupRedis(eventGenerator.getCampaigns(), configuration.get("redis"));

  // Connect the Ports in the Operators
  dag.addStream("filterTuples", eventGenerator.out, filterTuples.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("filterFields", filterTuples.output, filterFields.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("redisJoin", filterFields.output, redisJoin.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
  dag.addStream("output", redisJoin.output, campaignProcessor.input);

  dag.setInputPortAttribute(filterTuples.input, Context.PortContext.PARTITION_PARALLEL, true);
  dag.setInputPortAttribute(filterFields.input, Context.PortContext.PARTITION_PARALLEL, true);
  dag.setInputPortAttribute(redisJoin.input, Context.PortContext.PARTITION_PARALLEL, true);

  dag.setAttribute(eventGenerator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<EventGenerator>(8));
  dag.setAttribute(campaignProcessor, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<CampaignProcessor>(8));
}