com.datatorrent.api.Context.OperatorContext Java Examples

The following examples show how to use com.datatorrent.api.Context.OperatorContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SourceModule.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void activate(OperatorContext ctx)
{
  for (int i = 0; i < testNum; i++) {
    HashMap<String, Integer> dataMapa = new HashMap<String, Integer>();
    dataMapa.put("a", 2);
    holdingBuffer.add(dataMapa.toString().getBytes());

    HashMap<String, Integer> dataMapb = new HashMap<String, Integer>();
    dataMapb.put("b", 20);
    holdingBuffer.add(dataMapb.toString().getBytes());

    HashMap<String, Integer> dataMapc = new HashMap<String, Integer>();
    dataMapc.put("c", 1000);
    holdingBuffer.add(dataMapc.toString().getBytes());
  }
}
 
Example #2
Source File: InputOperatorTest.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Override
public void activate(OperatorContext ctx)
{
  dataGeneratorThread = new Thread("Integer Emitter")
  {
    @Override
    @SuppressWarnings("SleepWhileInLoop")
    public void run()
    {
      try {
        int i = 0;
        while (dataGeneratorThread != null) {
          (i % 2 == 0 ? evenBuffer : oddBuffer).put(i++);
          Thread.sleep(20);
        }
      } catch (InterruptedException ie) {
        // break out
      }
    }
  };
  dataGeneratorThread.start();
}
 
Example #3
Source File: KafkaExactlyOnceOutputOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void activate(OperatorContext ctx)
{
  dataGeneratorThread = new Thread("String Generator")
  {
    @Override
    public void run()
    {
      try {
        int i = 0;
        while (dataGeneratorThread != null && i < maxTuple) {
          stringBuffer.put((++i) + "###testString " + i);
        }
        stringBuffer.put((maxTuple + 1) + "###" + KafkaOperatorTestBase.END_TUPLE);
      } catch (InterruptedException ie) {
        //
      }
    }
  };
  dataGeneratorThread.start();
}
 
Example #4
Source File: ProcessingModeTests.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
public void testLinearInlineOperatorsRecovery() throws Exception
{
  RecoverableInputOperator.initGenTuples();
  CollectorOperator.collection.clear();
  CollectorOperator.duplicates.clear();

  dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
  dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
  dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
  RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
  rip.setMaximumTuples(maxTuples);
  rip.setSimulateFailure(true);

  CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
  cm.setSimulateFailure(true);
  dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);

  dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);

  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.run();
}
 
Example #5
Source File: AbstractFlumeInputOperator.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings({"unchecked"})
public void activate(OperatorContext ctx)
{
  if (connectionSpecs.length == 0) {
    logger.info("Discovered zero FlumeSink");
  } else if (connectionSpecs.length == 1) {
    for (String connectAddresse: connectionSpecs) {
      logger.debug("Connection spec is {}", connectAddresse);
      String[] parts = connectAddresse.split(":");
      eventloop.connect(new InetSocketAddress(parts[1], Integer.parseInt(parts[2])), client = new Client(parts[0]));
    }
  } else {
    throw new IllegalArgumentException(
        String.format("A physical %s operator cannot connect to more than 1 addresses!",
            this.getClass().getSimpleName()));
  }

  context = ctx;
}
 
Example #6
Source File: AbstractJMSInputOperator.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * Implement ActivationListener Interface.
 * @param ctx
 */
@Override
public void activate(OperatorContext ctx)
{
  try {
    super.createConnection();
    replyProducer = getSession().createProducer(null);

    consumer = (isDurable() && isTopic()) ?
        getSession().createDurableSubscriber((Topic)getDestination(), consumerName) :
        getSession().createConsumer(getDestination());
    consumer.setMessageListener(this);
  } catch (JMSException ex) {
    throw new RuntimeException(ex);
  }
}
 
Example #7
Source File: StreamingContainerTest.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Test
public void testCommitted() throws IOException, ClassNotFoundException
{
  LogicalPlan lp = new LogicalPlan();
  String workingDir = new File("target/testCommitted").getAbsolutePath();
  lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
  String opName = "CommitAwareOperatorTestCommit";
  lp.addOperator(opName, new CommitAwareOperator());

  StramLocalCluster lc = new StramLocalCluster(lp);
  lc.run(5000);

  /* this is not foolproof but some insurance is better than nothing */
  Assert.assertTrue("No Committed Windows", committedWindowIds.contains(opName));
}
 
Example #8
Source File: FSWindowDataManagerTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Test
public void testSave() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair = createManagerAndContextFor(1);
  pair.second.setup(pair.first);
  Map<Integer, String> data = Maps.newHashMap();
  data.put(1, "one");
  data.put(2, "two");
  data.put(3, "three");
  pair.second.save(data, 1);

  pair.second.setup(pair.first);
  @SuppressWarnings("unchecked")
  Map<Integer, String> artifact = (Map<Integer, String>)pair.second.retrieve(1);
  Assert.assertEquals("dataOf1", data, artifact);
  pair.second.teardown();
}
 
Example #9
Source File: JavaScriptOperator.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  for (Map.Entry<String, Object> entry : serializableBindings.entrySet()) {
    scriptBindings.put(entry.getKey(), entry.getValue());
  }
  this.scriptContext.setBindings(scriptBindings, ScriptContext.ENGINE_SCOPE);
  engine.setContext(this.scriptContext);
  try {
    for (String s : setupScripts) {
      engine.eval(s, this.scriptContext);
    }
  } catch (ScriptException ex) {
    throw new RuntimeException(ex);
  }
}
 
Example #10
Source File: AbstractKafkaInputOperator.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  logger.debug("consumer {} topic {} cacheSize {}", consumer, consumer.getTopic(), consumer.getCacheSize());
  consumer.create();
  // reset the offsets to checkpointed one
  if (consumer instanceof SimpleKafkaConsumer && !offsetStats.isEmpty()) {
    Map<KafkaPartition, Long> currentOffsets = new HashMap<>();
    // Increment the offsets and set it to consumer
    for (Map.Entry<KafkaPartition, Long> e: offsetStats.entrySet()) {
      currentOffsets.put(e.getKey(), e.getValue() + 1);
    }
    ((SimpleKafkaConsumer)consumer).resetOffset(currentOffsets);
  }
  this.context = context;
  operatorId = context.getId();
  if (consumer instanceof HighlevelKafkaConsumer && !(windowDataManager instanceof WindowDataManager.NoopWindowDataManager)) {
    throw new RuntimeException("Idempotency is not supported for High Level Kafka Consumer");
  }
  windowDataManager.setup(context);
}
 
Example #11
Source File: ManagedStateBenchmarkApp.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  TestStatsListener sl = new TestStatsListener();
  sl.adjustRate = conf.getBoolean("dt.ManagedStateBenchmark.adjustRate", false);
  TestGenerator gen = dag.addOperator("Generator", new TestGenerator());
  gen.setRange(timeRange);
  dag.setAttribute(gen, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)sl));

  storeOperator = new StoreOperator();
  storeOperator.setStore(createStore(conf));
  storeOperator.setTimeRange(timeRange);
  storeOperator = dag.addOperator("Store", storeOperator);

  dag.setAttribute(storeOperator, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)sl));

  dag.addStream("Events", gen.data, storeOperator.input).setLocality(Locality.CONTAINER_LOCAL);
}
 
Example #12
Source File: Slider.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  OutputPort<?> unifierOutputPort = getOutputPort();
  unifierOutputPort.setSink(
      new Sink<Object>()
      {
        @Override
        public void put(Object tuple)
        {
          outputPort.emit(tuple);
        }

        @Override
        public int getCount(boolean reset)
        {
          return 0;
        }
      }
  );
  unifier.setup(context);
  spinMillis = context.getValue(OperatorContext.SPIN_MILLIS);
}
 
Example #13
Source File: FilteredEventClassifier.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  noweight = new ArrayList<Integer>();
  for (int i = 0; i < keys.size(); i++) {
    noweight.add(100); // Even distribution
    total_weight += 100;
  }
  noweight.add(total_weight);
  if (pass_filter > total_filter) {
    throw new IllegalArgumentException(String.format("Pass filter (%d) cannot be >= Total filter (%d)", pass_filter, total_filter));
  }
}
 
Example #14
Source File: AbstractKinesisInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
/**
 * Implement Component Interface.
 *
 * @param context
 */
@Override
public void setup(OperatorContext context)
{
  this.context = context;
  try {
    KinesisUtil.getInstance().createKinesisClient(accessKey, secretKey, endPoint);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  consumer.create();
  operatorId = context.getId();
  windowDataManager.setup(context);
  shardPosition.clear();
}
 
Example #15
Source File: YahooFinanceCSVInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  url = prepareURL();
  client = new HttpClient();
  method = new GetMethod(url);
  DefaultHttpParams.getDefaultParams().setParameter("http.protocol.cookie-policy", CookiePolicy.BROWSER_COMPATIBILITY);
}
 
Example #16
Source File: AbstractJdbcPollInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void activate(OperatorContext context)
{
  long largestRecoveryWindow = windowManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID
      || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) > largestRecoveryWindow) {
    initializePreparedStatement();
    schedulePollTask();
  }
}
 
Example #17
Source File: PhysicalPlanTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
/**
 * Test that internally all stats listeners are handled through StatsListenerWithContext.
 * Following cases are tested
 *
 * Operator implementing StatsListener
 * Operator implementing StatsListenerWithContext
 * Operator with STATS_LISTENERS attribute set to StatsListener
 * Operator with STATS_LISTENERS attribute set to StatsListenerWithContext
 */
@Test
public void testStatsListenerContextWrappers()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  StatsListenerOperator o1 = dag.addOperator("o1", new StatsListenerOperator());
  GenericTestOperator o2 = dag.addOperator("o2", new GenericTestOperator());
  dag.setAttribute(o2, OperatorContext.STATS_LISTENERS,
      Lists.<StatsListener>newArrayList(mock(StatsListener.class)));

  GenericTestOperator o3 = dag.addOperator("o3", new GenericTestOperator());
  dag.setAttribute(o3, OperatorContext.STATS_LISTENERS,
      Lists.<StatsListener>newArrayList(mock(StatsListenerWithContext.class)));

  StatsListenerOperatorOld o4 = dag.addOperator("o4", new StatsListenerOperatorOld());

  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());

  PTOperator p1 = plan.getOperators(dag.getMeta(o1)).get(0);
  StatsListener l = p1.statsListeners.get(0);
  Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);

  PTOperator p2 = plan.getOperators(dag.getMeta(o2)).get(0);
  l = p1.statsListeners.get(0);
  Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);

  PTOperator p3 = plan.getOperators(dag.getMeta(o3)).get(0);
  l = p1.statsListeners.get(0);
  Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);

  PTOperator p4 = plan.getOperators(dag.getMeta(o4)).get(0);
  l = p1.statsListeners.get(0);
  Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
}
 
Example #18
Source File: AbstractFlumeInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void handleIdleTime()
{
  idleCounter++;
  try {
    sleep(context.getValue(OperatorContext.SPIN_MILLIS));
  } catch (InterruptedException ex) {
    throw new RuntimeException(ex);
  }
}
 
Example #19
Source File: OiOStreamTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void validatePositiveOiOiOdiamondWithCores()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());

  plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);

  plan.setOperatorAttribute(inputOperator, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator1, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator2, OperatorContext.VCORES, 2);
  plan.setOperatorAttribute(intermediateOperator3, OperatorContext.VCORES, 3);
  plan.setOperatorAttribute(intermediateOperator4, OperatorContext.VCORES, 5);
  plan.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  try {
    plan.validate();
    Assert.assertTrue("OiOiO extended diamond validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIOIO extended diamond validation");
  }
  PhysicalPlan physicalPlan = new PhysicalPlan(plan, new TestPlanContext());
  Assert.assertTrue("number of containers", 1 == physicalPlan.getContainers().size());
  Assert.assertTrue("number of vcores " + physicalPlan.getContainers().get(0).getRequiredVCores(), 5 == physicalPlan.getContainers().get(0).getRequiredVCores());
}
 
Example #20
Source File: FSWindowDataManagerTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Test
public void testRecovery() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair1 = createManagerAndContextFor(1);
  Pair<Context.OperatorContext, FSWindowDataManager> pair2 = createManagerAndContextFor(2);

  pair1.second.setup(pair1.first);
  pair2.second.setup(pair2.first);

  Map<Integer, String> dataOf1 = Maps.newHashMap();
  dataOf1.put(1, "one");
  dataOf1.put(2, "two");
  dataOf1.put(3, "three");

  Map<Integer, String> dataOf2 = Maps.newHashMap();
  dataOf2.put(4, "four");
  dataOf2.put(5, "five");
  dataOf2.put(6, "six");

  pair1.second.save(dataOf1, 1);
  pair2.second.save(dataOf2, 2);

  pair1.second.setup(pair1.first);
  Assert.assertEquals("largest recovery window", 1, pair1.second.getLargestCompletedWindow());

  pair2.second.setup(pair2.first);
  Assert.assertEquals("largest recovery window", 2, pair2.second.getLargestCompletedWindow());

  pair1.second.teardown();
  pair2.second.teardown();

  WindowDataManager manager = pair1.second.partition(1, Sets.newHashSet(2)).get(0);
  manager.setup(pair1.first);
  Assert.assertEquals("largest recovery window", 1, manager.getLargestCompletedWindow());
  manager.teardown();
}
 
Example #21
Source File: AbstractFlumeInputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void connected()
{
  super.connected();

  byte[] address;
  synchronized (recoveryAddresses) {
    if (recoveryAddresses.size() > 0) {
      address = recoveryAddresses.get(recoveryAddresses.size() - 1).address;
    } else {
      address = new byte[8];
    }
  }

  int len = 1 /* for the message type SEEK */
      + 8 /* for the address */
      + 8 /* for storing the current time stamp*/;

  byte[] array = new byte[len];
  array[0] = Server.Command.SEEK.getOrdinal();
  System.arraycopy(address, 0, array, 1, 8);
  Server.writeLong(array, 9, System.currentTimeMillis());
  write(array);

  connected = true;
  ConnectionStatus connectionStatus = new ConnectionStatus();
  connectionStatus.connected = true;
  connectionStatus.spec = connectionSpecs[0];
  OperatorContext ctx = context;
  synchronized (ctx) {
    logger.debug("{} Submitting ConnectionStatus = {}", AbstractFlumeInputOperator.this, connectionStatus);
    context.setCounters(connectionStatus);
  }
}
 
Example #22
Source File: JMSObjectInputOperatorTest.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
protected void starting(Description description)
{
  testBase = new JMSTestBase();
  try {
    testBase.beforTest();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  String methodName = description.getMethodName();
  String className = description.getClassName();
  baseDir = "target/" + className + "/" + methodName;

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(Context.OperatorContext.SPIN_MILLIS, 500);
  attributeMap.put(Context.DAGContext.APPLICATION_PATH, baseDir);

  context = mockOperatorContext(1, attributeMap);
  operator = new JMSObjectInputOperator();
  operator.setSubject("TEST.FOO");
  operator.getConnectionFactoryProperties().put(JMSTestBase.AMQ_BROKER_URL, "vm://localhost");

  sink = new CollectorTestSink<Object>();
  operator.output.setSink(sink);
  operator.setup(context);
  operator.activate(context);
}
 
Example #23
Source File: InputOperatorTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testSomeMethod() throws Exception
{
  LogicalPlan dag = new LogicalPlan();
  String testWorkDir = new File("target").getAbsolutePath();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir, null));
  EvenOddIntegerGeneratorInputOperator generator = dag.addOperator("NumberGenerator", EvenOddIntegerGeneratorInputOperator.class);
  final CollectorModule<Number> collector = dag.addOperator("NumberCollector", new CollectorModule<Number>());

  dag.addStream("EvenIntegers", generator.even, collector.even).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("OddIntegers", generator.odd, collector.odd).setLocality(Locality.CONTAINER_LOCAL);

  final StramLocalCluster lc = new StramLocalCluster(dag);
  lc.setHeartbeatMonitoringEnabled(false);

  lc.runAsync();
  WaitCondition c = new WaitCondition()
  {
    @Override
    public boolean isComplete()
    {
      return tupleCount.get() > 2;
    }
  };
  StramTestSupport.awaitCompletion(c, 2000);

  lc.shutdown();

  Assert.assertEquals("Collections size", 2, collections.size());
  Assert.assertFalse("Zero tuple count", collections.get(collector.even.id).isEmpty() && collections.get(collector.odd.id).isEmpty());
  Assert.assertTrue("Tuple count", collections.get(collector.even.id).size() - collections.get(collector.odd.id).size() <= 1);
}
 
Example #24
Source File: HBaseCsvMappingPutOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  parseMapping();
  lineListReader = new CsvListReader(lineSr,
  CsvPreference.STANDARD_PREFERENCE);
}
 
Example #25
Source File: S3TupleOutputModule.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  FSRecordCompactionOperator<INPUT> s3compaction = dag.addOperator("S3Compaction", new FSRecordCompactionOperator<INPUT>());
  s3compaction.setConverter(getConverter());
  s3compaction.setMaxIdleWindows(maxIdleWindows);
  s3compaction.setMaxLength(maxLength);

  StatelessThroughputBasedPartitioner<FSRecordCompactionOperator<INPUT>> partitioner = new StatelessThroughputBasedPartitioner<FSRecordCompactionOperator<INPUT>>();
  partitioner.setMaximumEvents(maxTuplesPerSecPerPartition);
  partitioner.setMinimumEvents(minTuplesPerSecPerPartition);
  partitioner.setCooldownMillis(coolDownMillis);
  dag.setAttribute(s3compaction, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] {partitioner}));
  dag.setAttribute(s3compaction, OperatorContext.PARTITIONER, partitioner);

  S3Reconciler s3Reconciler = dag.addOperator("S3Reconciler", new S3Reconciler());
  s3Reconciler.setAccessKey(accessKey);
  s3Reconciler.setSecretKey(secretAccessKey);
  s3Reconciler.setBucketName(bucketName);
  if (region != null) {
    s3Reconciler.setRegion(region);
  }
  s3Reconciler.setDirectoryName(outputDirectoryPath);

  S3ReconcilerQueuePartitioner<S3Reconciler> reconcilerPartitioner = new S3ReconcilerQueuePartitioner<S3Reconciler>();
  reconcilerPartitioner.setCooldownMillis(coolDownMillis);
  reconcilerPartitioner.setMinPartitions(minS3UploadPartitions);
  reconcilerPartitioner.setMaxPartitions(maxS3UploadPartitions);
  reconcilerPartitioner.setMaxQueueSizePerPartition(maxQueueSizeS3Upload);

  dag.setAttribute(s3Reconciler, OperatorContext.STATS_LISTENERS,
      Arrays.asList(new StatsListener[] {reconcilerPartitioner}));
  dag.setAttribute(s3Reconciler, OperatorContext.PARTITIONER, reconcilerPartitioner);

  dag.addStream("write-to-s3", s3compaction.output, s3Reconciler.input);
  input.set(s3compaction.input);
  output.set(s3Reconciler.outputPort);
}
 
Example #26
Source File: SeedEventClassifier.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  if (s_start > s_end) {
    int temp = s_end;
    s_end = s_start;
    s_start = temp;
  }
  seed = s_start;
}
 
Example #27
Source File: RegexMatchMapOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  if (this.regex != null) {
    pattern = Pattern.compile(this.regex);
  }
}
 
Example #28
Source File: WebSocketOutputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  try {
    openConnection();
  } catch (Exception ex) {
    LOG.warn("Cannot establish connection:", ex);
  }
}
 
Example #29
Source File: SplunkTcpOutputOperator.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  tcpInput = (TcpInput)store.getService().getInputs().get(tcpPort);
  try {
    socket = tcpInput.attach();
    stream = new DataOutputStream(socket.getOutputStream());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #30
Source File: PhysicalPlan.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}