com.datatorrent.api.Attribute.AttributeMap Java Examples

public static OperatorContext mockOperatorContext(int id, final AttributeMap map)
{
  OperatorContext context = Mockito.mock(OperatorContext.class);
  Mockito.when(context.getId()).thenReturn(id);
  Mockito.when(context.getAttributes()).thenReturn(map);
  Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).sendMetrics(Mockito.<Collection<String>>any());
  Mockito.doThrow(new UnsupportedOperationException("not supported")).when(context).setCounters(Mockito.any());
  Mockito.when(context.getValue(Mockito.<Attribute>any())).thenAnswer(new Answer<Object>()
  {
    @Override
    public Object answer(InvocationOnMock invocation) throws Throwable
    {
      final Attribute key = (Attribute)invocation.getArguments()[0];
      Object value = map.get(key);
      if (value != null) {
        return value;
      }
      return key.defaultValue;
    }
  });
  Mockito.doNothing().when(context).setCounters(Mockito.any());
  Mockito.when(context.getWindowsFromCheckpoint()).thenReturn(0);
  return context;
}
 

/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 

private void checkAttributeValueSerializable(AttributeMap attributes, String context)
{
  StringBuilder sb = new StringBuilder();
  String delim = "";
  // Check all attributes got operator are serializable
  for (Entry<Attribute<?>, Object> entry : attributes.entrySet()) {
    if (entry.getValue() != null && !(entry.getValue() instanceof Serializable)) {
      sb.append(delim).append(entry.getKey().getSimpleName());
      delim = ", ";
    }
  }
  if (sb.length() > 0) {
    throw new ValidationException("Attribute value(s) for " + sb.toString() + " in " + context + " are not serializable");
  }
}
 

/**
 * Copy attribute from source attributeMap to destination attributeMap.
 *
 * @param dest  destination attribute map.
 * @param source source attribute map.
 */
private void copyAttributes(AttributeMap dest, AttributeMap source)
{
  for (Entry<Attribute<?>, ?> a : source.entrySet()) {
    dest.put((Attribute<Object>)a.getKey(), a.getValue());
  }
}
 

private OperatorMeta(String name, GenericOperator operator, Attribute.AttributeMap attributeMap)
{
  LOG.debug("Initializing {} as {}", name, operator.getClass().getName());
  this.operatorAnnotation = operator.getClass().getAnnotation(OperatorAnnotation.class);
  this.name = name;
  this.operator = operator;
  this.id = logicalOperatorSequencer.decrementAndGet();
  this.status = new LogicalOperatorStatus(name);
  this.attributes = attributeMap;
}
 

@Test
public void TestSplunkInputOperator()
{
  SplunkStore store = new SplunkStore();
  store.setHost(HOST);
  store.setPassword(PASSWORD);
  store.setPort(PORT);
  store.setUserName(USER_NAME);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  TestInputOperator inputOperator = new TestInputOperator();
  inputOperator.setStore(store);
  inputOperator.setEarliestTime("-1000h");
  inputOperator.setLatestTime("now");
  CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
  inputOperator.outputPort.setSink(sink);

  inputOperator.setup(context);
  inputOperator.beginWindow(0);
  inputOperator.emitTuples();
  inputOperator.endWindow();

  Assert.assertEquals("rows from splunk", 100, sink.collectedTuples.size());
}
 

protected void setupOperator(HBasePOJOPutOperator operator)
{
  configure(operator);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);

  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  operator.setup(context);
}
 

protected void setupOperators()
{
  TableInfo<HBaseFieldInfo> tableInfo = new TableInfo<HBaseFieldInfo>();

  tableInfo.setRowOrIdExpression("row");

  List<HBaseFieldInfo> fieldsInfo = new ArrayList<HBaseFieldInfo>();
  fieldsInfo.add( new HBaseFieldInfo( "name", "name", SupportType.STRING, "f0") );
  fieldsInfo.add( new HBaseFieldInfo( "age", "age", SupportType.INTEGER, "f1") );
  fieldsInfo.add( new HBaseFieldInfo( "address", "address", SupportType.STRING, "f1") );

  tableInfo.setFieldsInfo(fieldsInfo);

  hbaseInputOperator.setTableInfo(tableInfo);
  hbaseOutputOperator.setTableInfo(tableInfo);

  store = new HBaseStore();
  store.setTableName("test");
  store.setZookeeperQuorum("localhost");
  store.setZookeeperClientPort(2181);

  hbaseInputOperator.setStore(store);
  hbaseOutputOperator.setStore(store);

  OperatorContext context = mockOperatorContext(OPERATOR_ID, new AttributeMap.DefaultAttributeMap());
  hbaseInputOperator.setup(context);
  hbaseOutputOperator.setup(context);
}
 

/**
 *
 * @param id the value of id
 * @param name name of the operator
 * @param attributes the value of attributes
 * @param parentContext
 */
public OperatorContext(int id, @NotNull String name, AttributeMap attributes, Context parentContext)
{
  super(attributes, parentContext);
  this.lastProcessedWindowId = Stateless.WINDOW_ID;
  this.id = id;
  this.name = Preconditions.checkNotNull(name, "operator name");
  this.stateless = super.getValue(OperatorContext.STATELESS);
}
 

@Before
public void setupForTest()
{
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  context = mockOperatorContext(OPERATOR_ID, attributeMap);

  Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, TestPojo.class);
  tpc = new TestPortContext(portAttributes);
}
 

/**
 * Saves the yarn application id which can be used by create application
 * specific table/region in KeyValue sore.
 */
@Override
public void setApplicationAttributes(AttributeMap map)
{
  this.applicationId = map.get(DAGContext.APPLICATION_ID);
  getStore().setTableName(applicationId);
}
 

@Test
public void testAtleastOnce() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();

    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));
    thop.beginWindow(0);
    thop.input.process(t1);
    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;

    tuple = HBaseTestHelper
        .getHBaseTuple("row1", "colfam0", "street");

    Assert.assertNotNull("Tuple", tuple);
    Assert.assertEquals("Tuple row", tuple.getRow(), "row1");
    Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
    Assert.assertEquals("Tuple column name", tuple.getColName(),"street");
    Assert.assertEquals("Tuple column value", tuple.getColValue(),"ts");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testAtmostOnce1() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();
    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));
    thop.beginWindow(0);
    thop.input.process(t1);
    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;

    tuple = HBaseTestHelper.getHBaseTuple("row1", "colfam0", "street");

    Assert.assertNotNull("Tuple", tuple);
    Assert.assertEquals("Tuple row", tuple.getRow(), "row1");
    Assert.assertEquals("Tuple column family", tuple.getColFamily(),
        "colfam0");
    Assert.assertEquals("Tuple column name", tuple.getColName(),
        "street");
    Assert.assertEquals("Tuple column value", tuple.getColValue(),
        "ts");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testAtmostOnce2() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();
    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    thop.beginWindow(0);
    thop.input.process(t1);
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));

    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;
    HBaseTuple tuple2;

    tuple = HBaseTestHelper.getHBaseTuple("row1", "colfam0", "street");
    tuple2 = HBaseTestHelper.getHBaseTuple("row2", "colfam0", "city");
    Assert.assertNull("Tuple", tuple);
    Assert.assertNotNull("Tuple2", tuple2);
    Assert.assertEquals("Tuple row", tuple2.getRow(), "row2");
    Assert.assertEquals("Tuple column family", tuple2.getColFamily(),"colfam0");
    Assert.assertEquals("Tuple column name", tuple2.getColName(),"city");
    Assert.assertEquals("Tuple column value", tuple2.getColValue(),"tc");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testCassandraInputOperator()
{
  String query1 = "SELECT * FROM " + KEYSPACE + "." + "%t;";
  CassandraStore store = new CassandraStore();
  store.setNode(NODE);
  store.setKeyspace(KEYSPACE);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  TestInputOperator inputOperator = new TestInputOperator();
  inputOperator.setStore(store);
  inputOperator.setQuery(query1);
  inputOperator.setTablename(TABLE_NAME_INPUT);
  inputOperator.setPrimaryKeyColumn("id");

  List<FieldInfo> fieldInfos = Lists.newArrayList();
  fieldInfos.add(new FieldInfo("id", "id", null));
  fieldInfos.add(new FieldInfo("age", "age", null));
  fieldInfos.add(new FieldInfo("lastname", "lastname", null));
  inputOperator.setFieldInfos(fieldInfos);

  inputOperator.insertEventsInTable(30);
  CollectorTestSink<Object> sink = new CollectorTestSink<>();
  inputOperator.outputPort.setSink(sink);

  Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, TestInputPojo.class);
  TestPortContext tpc = new TestPortContext(portAttributes);

  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(0);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 30, sink.collectedTuples.size());
  ArrayList<Integer> listOfIDs = inputOperator.getIds();
  // Rows are not stored in the same order in cassandra table in which they are inserted.
  for (int i = 0; i < 10; i++) {
    TestInputPojo object = (TestInputPojo)sink.collectedTuples.get(i);
    Assert.assertTrue("id set in testpojo", listOfIDs.contains(object.getId()));
    Assert.assertEquals("name set in testpojo", inputOperator.getNames().get(object.getId()), object.getLastname());
    Assert.assertEquals("age set in testpojo", inputOperator.getAge().get(object.getId()).intValue(), object.getAge());
  }

  sink.clear();
  inputOperator.columnDataTypes.clear();

  String query2 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v;";
  inputOperator.setQuery(query2);
  inputOperator.setStartRow(10);
  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(1);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 26, sink.collectedTuples.size());

  sink.clear();
  inputOperator.columnDataTypes.clear();

  String query3 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v LIMIT %l;";
  inputOperator.setQuery(query3);
  inputOperator.setStartRow(1);
  inputOperator.setLimit(10);
  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(2);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 10, sink.collectedTuples.size());
}
 

static OperatorContext getOperatorContext(final String app_id)
{
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, app_id);
  return mockOperatorContext(OPERATOR_ID, attributeMap);
}
 

@Test
public void testMemsqlOutputOperator() throws Exception
{
  cleanDatabase();
  MemsqlStore memsqlStore = createStore(null, true);

  MemsqlPOJOOutputOperator outputOperator = new MemsqlPOJOOutputOperator();
  outputOperator.setStore(memsqlStore);
  outputOperator.setBatchSize(BATCH_SIZE);
  outputOperator.setTablename(FQ_TABLE);
  ArrayList<String> columns = new ArrayList<String>();
  columns.add(DATA_COLUMN1);
  columns.add(DATA_COLUMN2);
  outputOperator.setDataColumns(columns);
  ArrayList<String> expressions = new ArrayList<String>();
  expressions.add("getIntVal()");
  expressions.add("getStringVal()");
  outputOperator.setExpression(expressions);
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  outputOperator.setup(context);

  for (int wid = 0; wid < NUM_WINDOWS; wid++) {
    outputOperator.beginWindow(wid);
    innerObj.setIntVal(wid + 1);
    outputOperator.input.put(innerObj);

    outputOperator.endWindow();
  }

  outputOperator.teardown();

  memsqlStore.connect();

  int databaseSize;

  Statement statement = memsqlStore.getConnection().createStatement();
  ResultSet resultSet = statement.executeQuery("select count(*) from " + FQ_TABLE);
  resultSet.next();
  databaseSize = resultSet.getInt(1);

  memsqlStore.disconnect();

  Assert.assertEquals("Numer of tuples in database", DATABASE_SIZE, databaseSize);
}
 

@Test
public void testHiveInsertMapOperator() throws SQLException, TException
{
  HiveStore hiveStore = createStore(null);
  hiveStore.setFilepath(testdir);
  ArrayList<String> hivePartitionColumns = new ArrayList<String>();
  hivePartitionColumns.add("dt");
  hiveInitializeMapDatabase(createStore(null));
  HiveOperator hiveOperator = new HiveOperator();
  hiveOperator.setStore(hiveStore);
  hiveOperator.setTablename(tablemap);
  hiveOperator.setHivePartitionColumns(hivePartitionColumns);

  FSRollingMapTestImpl fsRolling = new FSRollingMapTestImpl();
  fsRolling.setFilePath(testdir);
  short permission = 511;
  fsRolling.setFilePermission(permission);
  fsRolling.setAlwaysWriteToTmp(false);
  fsRolling.setMaxLength(128);
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  fsRolling.setup(context);
  hiveOperator.setup(context);
  HashMap<String, Object> map = new HashMap<String, Object>();
  FilePartitionMapping mapping1 = new FilePartitionMapping();
  FilePartitionMapping mapping2 = new FilePartitionMapping();
  ArrayList<String> partitions1 = new ArrayList<String>();
  partitions1.add("2014-12-10");
  mapping1.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-10" + "/" + "0-transaction.out.part.0");
  mapping1.setPartition(partitions1);
  ArrayList<String> partitions2 = new ArrayList<String>();
  partitions2.add("2014-12-11");
  mapping2.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-11" + "/" + "0-transaction.out.part.0");
  mapping2.setPartition(partitions2);
  for (int wid = 0; wid < NUM_WINDOWS; wid++) {
    fsRolling.beginWindow(wid);
    for (int tupleCounter = 0; tupleCounter < BLAST_SIZE; tupleCounter++) {
      map.put(2014 - 12 - 10 + "", 2014 - 12 - 10);
      fsRolling.input.put(map);
      map.clear();
    }

    if (wid == 7) {
      fsRolling.committed(wid - 1);
      hiveOperator.processTuple(mapping1);
      hiveOperator.processTuple(mapping2);
    }

    fsRolling.endWindow();
  }

  fsRolling.teardown();

  hiveStore.connect();

  client.execute("select * from " + tablemap + " where dt='2014-12-10'");
  List<String> recordsInDatePartition1 = client.fetchAll();

  client.execute("drop table " + tablemap);
  hiveStore.disconnect();

  Assert.assertEquals(13, recordsInDatePartition1.size());
  for (int i = 0; i < recordsInDatePartition1.size(); i++) {
    LOG.debug("records in first date partition are {}", recordsInDatePartition1.get(i));
    /*An array containing partition and data is returned as a string record, hence we need to upcast it to an object first
     and then downcast to a string in order to use in Assert.*/
    Object record = recordsInDatePartition1.get(i);
    Object[] records = (Object[])record;
    Assert.assertEquals("2014-12-10", records[1]);
  }

}
 

public TestStatsOperatorContext(int id, String name, AttributeMap attributes, Context parentContext)
{
  super(id, name, attributes, parentContext);
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return null;
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return attributes;
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return attributes;
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return attributes;
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return attributes;
}
 

@Override
public AttributeMap getAttributes()
{
  return this;
}
 

public PortContext(AttributeMap attributes, Context parentContext)
{
  super(attributes, parentContext);
}
 

@Override
public AttributeMap getAttributes()
{
  return contextAttributes;
}
 

@Override
public AttributeMap getAttributes()
{
  return contextAttributes;
}
 

@Override
public AttributeMap getAttributes()
{
  return contextAttributes;
}
 

public StreamingContainerContext(AttributeMap map, Context parentContext)
{
  super(map, parentContext);
}