Java Code Examples for com.datatorrent.api.Context#OperatorContext

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);

  CacheStore primaryCache = new CacheStore();

  // set expiration to one day.
  primaryCache.setEntryExpiryDurationInMillis(cacheExpirationInterval);
  primaryCache.setCacheCleanupInMillis(cacheCleanupInterval);
  primaryCache.setEntryExpiryStrategy(expiryType);
  primaryCache.setMaxCacheSize(cacheSize);

  cacheManager.setPrimary(primaryCache);
  cacheManager.setBackup(store);
}
 

@Override
public void setup(Context.OperatorContext context)
{
  AggregatorRegistry.DEFAULT_AGGREGATOR_REGISTRY.setup();

  schema = new DimensionalSchema(new DimensionalConfigurationSchema(eventSchemaJSON,
                                                            AggregatorRegistry.DEFAULT_AGGREGATOR_REGISTRY));

  maxProductId = 100;
  maxCustomerId = 30;
  maxChannelId = schema.getDimensionalConfigurationSchema().getKeysToEnumValuesList().get(KEY_CHANNEL).size();
  maxRegionId = schema.getDimensionalConfigurationSchema().getKeysToEnumValuesList().get(KEY_REGION).size();

  tuplesPerCurrentWindow = maxTuplesPerWindow;
  generateDiscounts();
  generateRegionalTax();
  initializeDataGenerators();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  currentWindowRecoveryState = Lists.newLinkedList();
  if (referenceTimes == null) {
    referenceTimes = new ConcurrentHashMap<>();
  }
  scanner.setup(context);
  windowDataManager.setup(context);
  super.setup(context);

  long largestRecoveryWindow = windowDataManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) >
      largestRecoveryWindow) {
    scanner.startScanning(Collections.unmodifiableMap(referenceTimes));
  }
}
 

@Override
public void setup(Context.OperatorContext context)
{
  this.client = siteToSiteBuilder.build();
  this.operatorContextId = context.getId();
  this.currentWindowTuples = new ArrayList<>();
  this.recoveredTuples = new ArrayList<>();
  this.windowDataManager.setup(context);
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  SlicedDirectoryScanner sds = (SlicedDirectoryScanner)getScanner();
  LOG.info("setup: directory = {}; scanner: " +
           "(startIndex = {}, endIndex = {}, pIndex = {})", getDirectory(),
           sds.getStartIndex(), sds.getEndIndex(), sds.getpIndex());

  pauseTime = context.getValue(Context.OperatorContext.SPIN_MILLIS);

  if (null != filePathStr) {      // restarting from checkpoint
    filePath = new Path(filePathStr);
  }
}
 

@Override
public void setup(Context.OperatorContext context)
{
  applicationName = context.getValue(Context.DAGContext.APPLICATION_NAME);
  consumerWrapper.create(this);
  metrics = new KafkaMetrics(metricsRefreshInterval);
  windowDataManager.setup(context);
  operatorId = context.getId();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  // create file name for this partition by appending the operator id to
  // the base name
  //
  long id = context.getId();
  fName = fileName + "_p" + id;
  super.setup(context);
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  startingTime = System.currentTimeMillis();
  watermarkTime = System.currentTimeMillis() + 10000;
  i = 1;
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  try {
    updateCommand = store.getConnection().prepareStatement(getUpdateCommand());
  } catch (SQLException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void setup(Context.OperatorContext context)
{
  Configuration configuration = new Configuration();
  appPath = new Path(context.getValue(DAG.APPLICATION_PATH) + Path.SEPARATOR + recoveryPath);

  try {
    storageAgent = new FSStorageAgent(appPath.toString(), configuration);

    fs = FileSystem.newInstance(appPath.toUri(), configuration);

    if (fs.exists(appPath)) {
      FileStatus[] fileStatuses = fs.listStatus(appPath);

      for (FileStatus operatorDirStatus : fileStatuses) {
        int operatorId = Integer.parseInt(operatorDirStatus.getPath().getName());

        for (FileStatus status : fs.listStatus(operatorDirStatus.getPath())) {
          String fileName = status.getPath().getName();
          if (fileName.endsWith(FSStorageAgent.TMP_FILE)) {
            continue;
          }
          long windowId = Long.parseLong(fileName, 16);
          replayState.put(windowId, operatorId);
          if (windowId > largestRecoveryWindow) {
            largestRecoveryWindow = windowId;
          }
        }
      }
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Test
public void testRecovery() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair1 = createManagerAndContextFor(1);
  Pair<Context.OperatorContext, FSWindowDataManager> pair2 = createManagerAndContextFor(2);

  pair1.second.setup(pair1.first);
  pair2.second.setup(pair2.first);

  Map<Integer, String> dataOf1 = Maps.newHashMap();
  dataOf1.put(1, "one");
  dataOf1.put(2, "two");
  dataOf1.put(3, "three");

  Map<Integer, String> dataOf2 = Maps.newHashMap();
  dataOf2.put(4, "four");
  dataOf2.put(5, "five");
  dataOf2.put(6, "six");

  pair1.second.save(dataOf1, 1);
  pair2.second.save(dataOf2, 2);

  pair1.second.setup(pair1.first);
  Assert.assertEquals("largest recovery window", 1, pair1.second.getLargestCompletedWindow());

  pair2.second.setup(pair2.first);
  Assert.assertEquals("largest recovery window", 2, pair2.second.getLargestCompletedWindow());

  pair1.second.teardown();
  pair2.second.teardown();

  WindowDataManager manager = pair1.second.partition(1, Sets.newHashSet(2)).get(0);
  manager.setup(pair1.first);
  Assert.assertEquals("largest recovery window", 1, manager.getLargestCompletedWindow());
  manager.teardown();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  try {
    connectable = new REngineConnectable();
    connectable.connect();
  } catch (IOException ioe) {
    log.error("Exception: ", ioe);
    DTThrowable.rethrow(ioe);
  }

}
 

@Override
public void setup(Context.OperatorContext context)
{
  operator.endWindow();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  operatorId = context.getId();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  done = false;
  result = new HashMap<>();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  operatorName = Preconditions.checkNotNull(context.getName(), "operator name");
  latch.countDown();
}
 

@Test
public void testDeleteDoesNotRemoveTmpFiles() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair1 = createManagerAndContextFor(1);
  pair1.second.setup(pair1.first);

  Pair<Context.OperatorContext, FSWindowDataManager> pair2 = createManagerAndContextFor(2);
  pair2.second.setup(pair2.first);

  Pair<Context.OperatorContext, FSWindowDataManager> pair3 = createManagerAndContextFor(3);
  pair3.second.setup(pair3.first);

  Map<Integer, String> dataOf1 = Maps.newHashMap();
  dataOf1.put(1, "one");
  dataOf1.put(2, "two");
  dataOf1.put(3, "three");

  Map<Integer, String> dataOf2 = Maps.newHashMap();
  dataOf2.put(4, "four");
  dataOf2.put(5, "five");
  dataOf2.put(6, "six");

  Map<Integer, String> dataOf3 = Maps.newHashMap();
  dataOf2.put(7, "seven");
  dataOf2.put(8, "eight");
  dataOf2.put(9, "nine");

  for (int i = 1; i <= 9; ++i) {
    pair1.second.save(dataOf1, i);
  }

  for (int i = 1; i <= 6; ++i) {
    pair2.second.save(dataOf2, i);
  }

  for (int i = 1; i <= 3; ++i) {
    pair3.second.save(dataOf3, i);
  }

  pair1.second.teardown();
  pair2.second.teardown();
  pair3.second.teardown();

  FSWindowDataManager fsManager = (FSWindowDataManager)pair1.second.partition(1, Sets.newHashSet(2, 3)).get(0);
  fsManager.setup(pair1.first);

  Assert.assertEquals("recovery window", 3, fsManager.getLargestCompletedWindow());

  Map<Integer, Object> artifacts = fsManager.retrieveAllPartitions(1);
  Assert.assertEquals("num artifacts", 3, artifacts.size());

  fsManager.committed(3);
  fsManager.teardown();

  testMeta.attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 3L);
  fsManager.setup(pair1.first);
  Assert.assertEquals("recovery window", Stateless.WINDOW_ID, fsManager.getLargestCompletedWindow());
  fsManager.teardown();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  readFunction();
}
 

@Override
public void setup(Context.OperatorContext context)
{
  TupleCount = 0;
}
 

@Override
public void setup(Context.OperatorContext context)
{
  throw new UnsupportedOperationException("not supported");
}