Java Code Examples for com.datatorrent.netlet.util.DTThrowable#rethrow()

@Override
public void storeAggregate()
{
  try {
    for (T tuple : tuples) {
      Mutation mutation = operationMutation(tuple);
      store.getBatchwriter().addMutation(mutation);
    }
    store.getBatchwriter().flush();

  } catch (MutationsRejectedException e) {
    logger.error("unable to write mutations", e);
    DTThrowable.rethrow(e);
  }
  tuples.clear();
}
 

@BeforeClass
public static void setup() {
  try {
    Class.forName(DB_DRIVER).newInstance();

    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
            + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
            + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
            + ")";
    stmt.executeUpdate(createMetaTable);

    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
            + " (ACCOUNT_NO INTEGER, NAME VARCHAR(255),AMOUNT INTEGER)";
    stmt.executeUpdate(createTable);

  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void endWindow()
{
  while (replay && scanCallsInCurrentWindow < recoveryState.numberOfScanCallsInWindow) {
    // If less keys got scanned in this window, scan till recovery offset
    scanKeysFromOffset();
    processTuples();
  }
  super.endWindow();
  recoveryState.scanOffsetAtBeginWindow = scanOffset;
  recoveryState.numberOfScanCallsInWindow = scanCallsInCurrentWindow;

  if (currentWindowId > getWindowDataManager().getLargestCompletedWindow()) {
    try {
      getWindowDataManager().save(recoveryState, currentWindowId);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 

@Override
public void setup(OperatorContext context)
{
  // Needed to setup idempotency storage manager in setter
  this.context = context;
  this.operatorContextId = context.getId();

  try {
    connFactory.setHost("localhost");
    connection = connFactory.newConnection();
    channel = connection.createChannel();
    channel.exchangeDeclare(exchange, "fanout");

    this.windowDataManager.setup(context);

  } catch (IOException ex) {
    logger.debug(ex.toString());
    DTThrowable.rethrow(ex);
  }
}
 

@Override
public void run()
{
  running = true;
  try {
    while (running) {
      if (trigger || (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis)) {
        trigger = false;
        for (String afile : files) {
          scan(new Path(afile), null);
        }
        scanComplete();
      } else {
        Thread.sleep(sleepMillis);
      }
    }
  } catch (Throwable throwable) {
    LOG.error("service", throwable);
    running = false;
    atomicThrowable.set(throwable);
    DTThrowable.rethrow(throwable);
  }
}
 

@BeforeClass
public static void setup()
{
  try {
    cluster = Cluster.builder().addContactPoint(NODE).build();
    session = cluster.connect(KEYSPACE);

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "."
        + CassandraTransactionalStore.DEFAULT_META_TABLE + " ( " + CassandraTransactionalStore.DEFAULT_APP_ID_COL
        + " TEXT, " + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT, "
        + CassandraTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT, " + "PRIMARY KEY ("
        + CassandraTransactionalStore.DEFAULT_APP_ID_COL + ", " + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL
        + ") " + ");";
    session.execute(createMetaTable);
    String createTable = "CREATE TABLE IF NOT EXISTS "
        + KEYSPACE
        + "."
        + TABLE_NAME
        + " (id uuid PRIMARY KEY,age int,lastname text,test boolean,floatvalue float,doubleValue double,set1 set<int>,list1 list<int>,map1 map<text,int>,last_visited timestamp);";
    session.execute(createTable);
  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public ConnectionFactory buildConnectionFactory()
{
  ConnectionFactory cf;
  try {
    if (connectionFactoryClass != null) {
      @SuppressWarnings("unchecked")
      Class<ConnectionFactory> clazz = (Class<ConnectionFactory>)Class.forName(connectionFactoryClass);
      cf = clazz.newInstance();
    } else {
      cf = new org.apache.activemq.ActiveMQConnectionFactory();
    }
    BeanUtils.populate(cf, connectionFactoryProperties);
    logger.debug("creation successful.");
    return cf;
  } catch (Exception e) {
    DTThrowable.rethrow(e);
    return null;  // previous rethrow makes this redundant, but compiler doesn't know...
  }
}
 

@Override
public void committed(long windowId)
{
  if (initialOffset == InitialOffset.LATEST || initialOffset == InitialOffset.EARLIEST) {
    return;
  }
  //ask kafka consumer wrapper to store the committed offsets
  for (Iterator<Pair<Long, Map<AbstractKafkaPartitioner.PartitionMeta, Long>>> iter =
      offsetHistory.iterator(); iter.hasNext(); ) {
    Pair<Long, Map<AbstractKafkaPartitioner.PartitionMeta, Long>> item = iter.next();
    if (item.getLeft() <= windowId) {
      if (item.getLeft() == windowId) {
        consumerWrapper.commitOffsets(item.getRight());
      }
      iter.remove();
    }
  }
  if (isIdempotent()) {
    try {
      windowDataManager.committed(windowId);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 

/**
 * This creates a snapshot schema from the specified json.
 * @param schemaJSON The JSON specifying the snapshot schema.
 */
public SnapshotSchema(String schemaJSON)
{
  setSchema(schemaJSON);

  try {
    initialize();
  } catch (Exception ex) {
    DTThrowable.rethrow(ex);
  }
}
 

public void setup(com.datatorrent.api.Context.OperatorContext context)
{
  store = new ElasticSearchPercolatorStore(hostName, port);
  try {
    store.connect();
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void teardown()
{
  try {
    csvReader.close();
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void setup(Context.OperatorContext context)
{
  super.setup(context);
  try {
    connectable = new REngineConnectable();
    connectable.connect();
  } catch (IOException ioe) {
    log.error("Exception: ", ioe);
    DTThrowable.rethrow(ioe);
  }

}
 

@Override
public void process(byte[] message)
{
  String inputString = new String(message);
  try {
    JSONObject jSONObject = new JSONObject(inputString);

    // output JSONObject
    outputJsonObject.emit(jSONObject);

    // output map retaining the tree structure from json
    if (outputMap.isConnected()) {
      Iterator<String> iterator = jSONObject.keys();
      HashMap<String, Object> map = new HashMap<String, Object>();
      while (iterator.hasNext()) {
        String key = iterator.next();
        map.put(key, jSONObject.getString(key));
      }
      outputMap.emit(map);
    }

    // output map as flat key value pairs
    if (outputFlatMap.isConnected()) {
      HashMap<String, Object> flatMap = new HashMap<String, Object>();
      getFlatMap(jSONObject, flatMap, null);
      outputFlatMap.emit(flatMap);
    }

  } catch (Throwable ex) {
    DTThrowable.rethrow(ex);
  }
}
 

public static void checkOutput(int fileCount, String baseFilePath, String expectedOutput, boolean checkTmp)
{
  if (fileCount >= 0) {
    baseFilePath += "." + fileCount;
  }

  File file = new File(baseFilePath);

  if (!file.exists()) {
    String[] extensions = {"tmp"};
    Collection<File> tmpFiles = FileUtils.listFiles(file.getParentFile(), extensions, false);
    for (File tmpFile : tmpFiles) {
      if (file.getPath().startsWith(baseFilePath)) {
        file = tmpFile;
        break;
      }
    }
  }

  String fileContents = null;

  try {
    fileContents = FileUtils.readFileToString(file);
  } catch (IOException ex) {
    DTThrowable.rethrow(ex);
  }

  Assert.assertEquals("Single file " + fileCount + " output contents", expectedOutput, fileContents);
}
 

@Override
public void setup(OperatorContext context)
{
  server = new Server(port);
  DefaultWebSocketServlet servlet = new DefaultWebSocketServlet();
  ServletHolder sh = new ServletHolder(servlet);
  ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
  contextHandler.addServlet(sh, extension);

  try {
    server.start();
  } catch (Exception ex) {
    DTThrowable.rethrow(ex);
  }
}
 

@Override
public void disconnect() throws IOException
{
  try {
    batchwriter.close();
  } catch (MutationsRejectedException e) {
    logger.error("mutation rejected during batchwriter close", e);
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void teardown()
{
  super.teardown();
  try {
    lineSr.close();
    lineListReader.close();
  } catch (IOException e) {
    logger.error("Cannot close the readers", e);
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void processTuple(T tuple, HTable table)
{
  Append append = operationAppend(tuple);
  try {
    table.append(append);
  } catch (IOException e) {
    logger.error("Could not append tuple", e);
    DTThrowable.rethrow(e);
  }

}
 

@Override
public void process(byte[] message)
{
  try {
    // Convert byte array JSON representation to HashMap
    Map<String, Object> tuple = reader.readValue(message);
    outputMap.emit(tuple);
  } catch (Throwable ex) {
    DTThrowable.rethrow(ex);
  }
}
 

@Override
public void emitTuples()
{
  if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
    return;
  }

  Throwable throwable;
  if ((throwable = scanner.atomicThrowable.get()) != null) {
    DTThrowable.rethrow(throwable);
  }
  if (blockMetadataIterator != null && blockCount < blocksThreshold) {
    emitBlockMetadata();
  }

  FileInfo fileInfo;
  while (blockCount < blocksThreshold && (fileInfo = scanner.pollFile()) != null) {

    currentWindowRecoveryState.add(fileInfo);
    try {
      FileMetadata fileMetadata = buildFileMetadata(fileInfo);
      filesMetadataOutput.emit(fileMetadata);
      fileCounters.getCounter(Counters.PROCESSED_FILES).increment();
      if (!fileMetadata.isDirectory()) {
        blockMetadataIterator = new BlockMetadataIterator(this, fileMetadata, blockSize);
        if (!emitBlockMetadata()) {
          //block threshold reached
          break;
        }
      }
      if (fileInfo.lastFileOfScan) {
        break;
      }
    } catch (IOException e) {
      throw new RuntimeException("creating metadata", e);
    }
  }
}