Java Code Examples for com.datatorrent.netlet.util.DTThrowable

The following examples show how to use com.datatorrent.netlet.util.DTThrowable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: attic-apex-malhar   Source File: FileSplitter.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void teardown()
{
  try {
    scanner.teardown();
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  } finally {
    try {
      fs.close();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
}
 
Example 2
Source Project: examples   Source File: ApplicationTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() {
  try {
    Class.forName(DB_DRIVER).newInstance();

    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
            + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
            + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
            + ")";
    stmt.executeUpdate(createMetaTable);

    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
            + " (ACCOUNT_NO INTEGER, NAME VARCHAR(255),AMOUNT INTEGER)";
    stmt.executeUpdate(createTable);

  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 3
Source Project: attic-apex-malhar   Source File: ApplicationTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup()
{
  try {
    Class.forName(DB_DRIVER).newInstance();

    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
        + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
        + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
        + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
        + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
        + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
        + ")";
    stmt.executeUpdate(createMetaTable);

    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
        + " (ACCOUNT_NO INTEGER, NAME VARCHAR(255),AMOUNT INTEGER)";
    stmt.executeUpdate(createTable);

  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 4
@Override
public void endWindow()
{
  // if replaying then nothing to do
  if (currentWindowId <= largestRecoveryWindowId) {
    return;
  }

  // if processing a new window then give sub-classes a chance to take action
  endNewWindow();

  // mark that we processed the window
  try {
    windowDataManager.save("processedWindow", currentWindowId);
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 5
@Override
public void beginWindow(long windowId)
{
  currentWindowId = windowId;

  // if the current window is now less than the largest window, then we need to replay data
  if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
    try {
      List<T> recoveredData =  (List<T>)this.windowDataManager.retrieve(windowId);
      if (recoveredData == null) {
        return;
      }

      // if we recovered tuples then load them to be processed by next call to emitTuples()
      recoveredTuples.addAll(recoveredData);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 
Example 6
/**
 * This executes the query to retrieve result from database.
 * It then converts each row into tuple and emit that into output port.
 */
@Override
public void emitTuples()
{
  Statement query = queryToRetrieveData();
  logger.debug(String.format("select statement: %s", query.toString()));
  RecordSet rs;
  try {
    rs = store.getClient().query(null, query);
    while (rs.next()) {
      Record rec = rs.getRecord();
      T tuple = getTuple(rec);
      outputPort.emit(tuple);
    }
  } catch (Exception ex) {
    store.disconnect();
    DTThrowable.rethrow(ex);
  }
}
 
Example 7
@Override
public void storeAggregate()
{
  try {
    for (T tuple : tuples) {
      Mutation mutation = operationMutation(tuple);
      store.getBatchwriter().addMutation(mutation);
    }
    store.getBatchwriter().flush();

  } catch (MutationsRejectedException e) {
    logger.error("unable to write mutations", e);
    DTThrowable.rethrow(e);
  }
  tuples.clear();
}
 
Example 8
Source Project: attic-apex-malhar   Source File: AccumuloWindowStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public long getCommittedWindowId(String appId, int operatorId)
{
  byte[] value = null;
  Authorizations auths = new Authorizations();
  Scanner scan = null;
  String columnKey = appId + "_" + operatorId + "_" + lastWindowColumnName;
  lastWindowColumnBytes = columnKey.getBytes();
  try {
    scan = connector.createScanner(tableName, auths);
  } catch (TableNotFoundException e) {
    logger.error("error getting committed window id", e);
    DTThrowable.rethrow(e);
  }
  scan.setRange(new Range(new Text(rowBytes)));
  scan.fetchColumn(new Text(columnFamilyBytes), new Text(lastWindowColumnBytes));
  for (Entry<Key, Value> entry : scan) {
    value = entry.getValue().get();
  }
  if (value != null) {
    long longval = toLong(value);
    return longval;
  }
  return -1;
}
 
Example 9
Source Project: attic-apex-malhar   Source File: AccumuloWindowStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void storeCommittedWindowId(String appId, int operatorId,long windowId)
{
  byte[] WindowIdBytes = toBytes(windowId);
  String columnKey = appId + "_" + operatorId + "_" + lastWindowColumnName;
  lastWindowColumnBytes = columnKey.getBytes();
  Mutation mutation = new Mutation(rowBytes);
  mutation.put(columnFamilyBytes, lastWindowColumnBytes, WindowIdBytes);
  try {
    batchwriter.addMutation(mutation);
    batchwriter.flush();
  } catch (MutationsRejectedException e) {
    logger.error("error getting committed window id", e);
    DTThrowable.rethrow(e);
  }
}
 
Example 10
Source Project: attic-apex-malhar   Source File: REngineConnectable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void connect() throws IOException
{
  try {

    rengine = REngine.getLastEngine();
    if (rengine == null) {
      // new R-engine
      rengine = REngine.engineForClass(rEngineClassName, args, callBacks, runREPL);
      log.info("Creating new Rengine");
    } else {
      log.info("Got last Rengine");
    }
  } catch (Exception exc) {
    log.error("Exception: ", exc);
    DTThrowable.rethrow(exc);
  }

}
 
Example 11
Source Project: attic-apex-malhar   Source File: FileSplitter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run()
{
  running = true;
  try {
    while (running) {
      if (trigger || (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis)) {
        trigger = false;
        for (String afile : files) {
          scan(new Path(afile), null);
        }
        scanComplete();
      } else {
        Thread.sleep(sleepMillis);
      }
    }
  } catch (Throwable throwable) {
    LOG.error("service", throwable);
    running = false;
    atomicThrowable.set(throwable);
    DTThrowable.rethrow(throwable);
  }
}
 
Example 12
private void replay(long windowId)
{
  try {
    // For first recovered window, offset is already part of recovery state.
    // So skip reading from idempotency manager
    if (!skipOffsetRecovery) {
      // Begin offset for this window is recovery offset stored for the last
      // window
      RecoveryState recoveryStateForLastWindow = (RecoveryState)getWindowDataManager().retrieve(windowId - 1);
      recoveryState.scanOffsetAtBeginWindow = recoveryStateForLastWindow.scanOffsetAtBeginWindow;
    }
    skipOffsetRecovery = false;
    RecoveryState recoveryStateForCurrentWindow = (RecoveryState)getWindowDataManager().retrieve(windowId);
    recoveryState.numberOfScanCallsInWindow = recoveryStateForCurrentWindow.numberOfScanCallsInWindow;
    if (recoveryState.scanOffsetAtBeginWindow != null) {
      scanOffset = recoveryState.scanOffsetAtBeginWindow;
    }
    replay = true;

  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 13
private void scanKeysFromOffset()
{
  if (!scanComplete) {
    if (replay && scanCallsInCurrentWindow >= recoveryState.numberOfScanCallsInWindow) {
      try {
        Thread.sleep(sleepTimeMillis);
      } catch (InterruptedException e) {
        DTThrowable.rethrow(e);
      }
      return;
    }

    ScanResult<String> result = store.ScanKeys(scanOffset, scanParameters);
    backupOffset = scanOffset;
    scanOffset = Integer.parseInt(result.getStringCursor());
    if (scanOffset == 0) {
      // Redis store returns 0 after all data is read
      scanComplete = true;

      // point scanOffset to the end in this case for reading any new tuples
      scanOffset = backupOffset + result.getResult().size();
    }
    keys = result.getResult();
  }
  scanCallsInCurrentWindow++;
}
 
Example 14
@Override
public void endWindow()
{
  while (replay && scanCallsInCurrentWindow < recoveryState.numberOfScanCallsInWindow) {
    // If less keys got scanned in this window, scan till recovery offset
    scanKeysFromOffset();
    processTuples();
  }
  super.endWindow();
  recoveryState.scanOffsetAtBeginWindow = scanOffset;
  recoveryState.numberOfScanCallsInWindow = scanCallsInCurrentWindow;

  if (currentWindowId > getWindowDataManager().getLargestCompletedWindow()) {
    try {
      getWindowDataManager().save(recoveryState, currentWindowId);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 
Example 15
Source Project: attic-apex-malhar   Source File: RedisPOJOInputOperator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void processTuples()
{
  for (String key : keys) {
    if (store.getType(key).equals("hash")) {
      Map<String, String> mapValue = store.getMap(key);
      if (isFirstTuple) {
        try {
          processFirstTuple(mapValue);
        } catch (ClassNotFoundException e) {
          DTThrowable.rethrow(e);
        }
      }
      isFirstTuple = false;
      outputPort.emit(new KeyValPair<String, Object>(key, convertMapToObject(mapValue)));
    }
  }
  keys.clear();
}
 
Example 16
@Override
protected void finished(Description description)
{
  try {
    // Clean up
    session.close();
    connection.close();
  } catch (JMSException ex) {
    DTThrowable.rethrow(ex);
  }
  operator.deactivate();
  operator.teardown();
  try {
    FileUtils.deleteDirectory(new File("target/" + description.getClassName()));
    testBase.afterTest();
  } catch (Exception e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 17
Source Project: attic-apex-malhar   Source File: CassandraStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a cluster object.
 */
public void buildCluster()
{
  try {
    if (protocolVersion != null && protocolVersion.length() != 0) {
      ProtocolVersion version = getCassandraProtocolVersion();
      cluster = Cluster.builder().addContactPoint(node).withCredentials(userName, password).withProtocolVersion(version).build();
    } else {
      cluster = Cluster.builder().addContactPoint(node).withCredentials(userName, password).build();
    }
  } catch (DriverException ex) {
    throw new RuntimeException("closing database resource", ex);
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  }
}
 
Example 18
Source Project: attic-apex-malhar   Source File: CassandraStore.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create connection with database.
 */
@Override
public void connect()
{
  try {
    if (cluster == null) {
      buildCluster();
    }
    session = cluster.connect();
    logger.debug("Cassandra connection Success");
  } catch (DriverException ex) {
    throw new RuntimeException("closing database resource", ex);
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  }
}
 
Example 19
public PercolateResponse percolate(String[] indexNames, String documentType, Object tuple)
{
  XContentBuilder docBuilder;
  try {

    docBuilder = XContentFactory.jsonBuilder().startObject();
    docBuilder.field("doc").startObject(); //This is needed to designate the document
    docBuilder.field("content", tuple);
    docBuilder.endObject(); //End of the doc field
    docBuilder.endObject();//End of the JSON root object

    return client.preparePercolate().setIndices(indexNames)
        .setDocumentType(documentType)
        .setSource(docBuilder)
        .execute().actionGet();
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
  return null;
}
 
Example 20
@Override
public void setup(OperatorContext context)
{
  // Needed to setup idempotency storage manager in setter
  this.context = context;
  this.operatorContextId = context.getId();

  try {
    connFactory.setHost("localhost");
    connection = connFactory.newConnection();
    channel = connection.createChannel();
    channel.exchangeDeclare(exchange, "fanout");

    this.windowDataManager.setup(context);

  } catch (IOException ex) {
    logger.debug(ex.toString());
    DTThrowable.rethrow(ex);
  }
}
 
Example 21
@SuppressWarnings("unchecked")
private void replay(long windowId)
{
  Map<Long, byte[]> recoveredData;
  try {
    recoveredData = (Map<Long, byte[]>)this.windowDataManager.retrieve(windowId);
    if (recoveredData == null) {
      return;
    }
    for (Entry<Long, byte[]>  recoveredEntry : recoveredData.entrySet()) {
      recoveredTags.add(recoveredEntry.getKey());
      emitTuple(recoveredEntry.getValue());
    }
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 22
@Override
public Scanner getScanner(Connector conn)
{
  Authorizations auths = new Authorizations();
  Scanner scan = null;
  try {
    scan = conn.createScanner(getStore().getTableName(), auths);
  } catch (TableNotFoundException e) {
    logger.error("table not found ");
    DTThrowable.rethrow(e);
  }
  scan.setRange(new Range());
  // scan.fetchColumnFamily("attributes");

  return scan;
}
 
Example 23
Source Project: attic-apex-malhar   Source File: AccumuloTestHelper.java    License: Apache License 2.0 6 votes vote down vote up
public static AccumuloTuple getAccumuloTuple(String row, String colFam,
    String colName)
{
  Authorizations auths = new Authorizations();

  Scanner scan = null;
  try {
    scan = con.createScanner("tab1", auths);
  } catch (TableNotFoundException e) {
    logger.error("error in test helper");
    DTThrowable.rethrow(e);
  }

  scan.setRange(new Range(new Text(row)));
  scan.fetchColumn(new Text(colFam), new Text(colName));
  // assuming only one row
  for (Entry<Key, Value> entry : scan) {
    AccumuloTuple tuple = new AccumuloTuple();
    tuple.setRow(entry.getKey().getRow().toString());
    tuple.setColFamily(entry.getKey().getColumnFamily().toString());
    tuple.setColName(entry.getKey().getColumnQualifier().toString());
    tuple.setColValue(entry.getValue().toString());
    return tuple;
  }
  return null;
}
 
Example 24
Source Project: attic-apex-malhar   Source File: JDBCLoaderTest.java    License: Apache License 2.0 6 votes vote down vote up
private void insertRecordsInTable()
{
  try {
    Statement stmt = dbloader.getConnection().createStatement();
    String tbName = dbloader.getTableName();

    for (int i = 0; i < id.length; i++) {
      String sql = "INSERT INTO " + tbName + " (ID,NAME,AGE,ADDRESS,SALARY) " +
          "VALUES (" + id[i] + ", '" + name[i] + "', " + age[i] + ", '" + address[i] + "', " + salary[i] + " );";
      stmt.executeUpdate(sql);
    }
  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }

}
 
Example 25
Source Project: attic-apex-malhar   Source File: CassandraOperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup()
{
  @SuppressWarnings("UnusedDeclaration")
  Class<?> clazz = org.codehaus.janino.CompilerFactory.class;
  try {
    cluster = Cluster.builder()
            .addContactPoint(NODE).build();
    session = cluster.connect(KEYSPACE);

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + CassandraTransactionalStore.DEFAULT_META_TABLE + " ( "
        + CassandraTransactionalStore.DEFAULT_APP_ID_COL + " TEXT, "
        + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT, "
        + CassandraTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT, "
        + "PRIMARY KEY (" + CassandraTransactionalStore.DEFAULT_APP_ID_COL + ", " + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL + ") "
        + ");";
    session.execute(createMetaTable);
    String createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE_NAME + " (id uuid PRIMARY KEY,age int,lastname text,test boolean,floatvalue float,doubleValue double,set1 set<int>,list1 list<int>,map1 map<text,int>,last_visited timestamp);";
    session.execute(createTable);
    createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE_NAME_INPUT + " (id int PRIMARY KEY,lastname text,age int);";
    session.execute(createTable);
  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 
Example 26
@Override
public byte[] getTuple(byte[] message)
{
  if (registry == null) {
    return message;
  }
  String inputString = new String(message);
  try {
    JSONObject jSONObject = new JSONObject(inputString);
    int typeId = registry.getIndex(LogstreamUtil.LOG_TYPE, routingKey);
    jSONObject.put(LogstreamUtil.LOG_TYPE, typeId);
    String outputString = jSONObject.toString();
    message = outputString.getBytes();
  }
  catch (Throwable ex) {
    DTThrowable.rethrow(ex);
  }

  return message;
}
 
Example 27
Source Project: attic-apex-malhar   Source File: LogstreamUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * returns the number value of the supplied object
 * @param value
 * @return
 */
public static Number extractNumber(Object value)
{
  NumberFormat numberFormat = NumberFormat.getInstance();
  if (value instanceof Number) {
    return (Number)value;
  }
  else if (value == null) {
    return new Long(0);
  }
  else {
    try {
      return numberFormat.parse(value.toString());
    }
    catch (ParseException ex) {
      DTThrowable.rethrow(ex);
    }
  }
  return new Long(0);
}
 
Example 28
Source Project: attic-apex-malhar   Source File: JdbcStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create connection with database using JDBC.
 */
@Override
public void connect()
{
  try {
    // This will load the JDBC driver, each DB has its own driver
    Class.forName(databaseDriver).newInstance();
    connection = DriverManager.getConnection(databaseUrl, connectionProperties);

    logger.debug("JDBC connection Success");
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  }
}
 
Example 29
Source Project: examples   Source File: JsonToMapConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void process(byte[] message)
{
  try {
    // Convert byte array JSON representation to HashMap
    Map<String, Object> tuple = reader.readValue(message);
    outputMap.emit(tuple);
  } catch (Throwable ex) {
    DTThrowable.rethrow(ex);
  }
}
 
Example 30
Source Project: streaming-benchmarks   Source File: FilterTuples.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void process(JSONObject jsonObject)
{
  try {
    if (  jsonObject.getString("event_type").equals("view") ) {
      output.emit(jsonObject);
    }
  } catch (JSONException e) {
    DTThrowable.wrapIfChecked(e);
  }
}