Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#getOperatorMeta()

The following examples show how to use com.datatorrent.stram.plan.logical.LogicalPlan#getOperatorMeta() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PhysicalPlan.java    From Bats with Apache License 2.0 5 votes vote down vote up 
public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap)
{
  for (AffinityRule rule : affinityRules) {
    if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
      for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
        for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
          OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));

          PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
          OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
          PMapping secondMapping = logicalToPTOperator.get(opMeta);
          for (PTOperator firstPtOperator : firstPMapping.partitions) {
            PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
            for (PTOperator secondPtOperator : secondMapping.partitions) {
              PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
              if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
                continue;
              }
              if (rule.isRelaxLocality()) {
                firstContainer.getPreferredAntiPrefs().add(secondContainer);
                secondContainer.getPreferredAntiPrefs().add(firstContainer);
              } else {
                firstContainer.getStrictAntiPrefs().add(secondContainer);
                secondContainer.getStrictAntiPrefs().add(firstContainer);
              }
            }
          }
        }
      }
    }
  }
}
Example 2
Source File: PhysicalPlan.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap)
{
  for (AffinityRule rule : affinityRules) {
    if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
      for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
        for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
          OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));

          PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
          OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
          PMapping secondMapping = logicalToPTOperator.get(opMeta);
          for (PTOperator firstPtOperator : firstPMapping.partitions) {
            PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
            for (PTOperator secondPtOperator : secondMapping.partitions) {
              PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
              if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
                continue;
              }
              if (rule.isRelaxLocality()) {
                firstContainer.getPreferredAntiPrefs().add(secondContainer);
                secondContainer.getPreferredAntiPrefs().add(firstContainer);
              } else {
                firstContainer.getStrictAntiPrefs().add(secondContainer);
                secondContainer.getStrictAntiPrefs().add(firstContainer);
              }
            }
          }
        }
      }
    }
  }
}
Example 3
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
private void validateOperatorParent(LogicalPlan dag, String operatorName, String parentModuleName)
{
  LogicalPlan.OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
  if (parentModuleName == null) {
    Assert.assertNull(operatorMeta.getModuleName());
  } else {
    Assert.assertTrue(parentModuleName.equals(operatorMeta.getModuleName()));
  }
}
Example 4
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
/**
 * Verify attributes populated on DummyOperator from Level1 module
 */
private void validateOperatorAttribute(LogicalPlan dag, String name, int memory)
{
  LogicalPlan.OperatorMeta oMeta = dag.getOperatorMeta(name);
  Attribute.AttributeMap attrs = oMeta.getAttributes();
  Assert.assertEquals((int)attrs.get(OperatorContext.MEMORY_MB), memory);
  Assert.assertEquals("Application window id is 2 ", (int)attrs.get(OperatorContext.APPLICATION_WINDOW_COUNT), 2);
  Assert.assertEquals("Locality host is host1", attrs.get(OperatorContext.LOCALITY_HOST), "host1");
  Assert.assertEquals(attrs.get(OperatorContext.PARTITIONER).getClass(), TestPartitioner.class);
  Assert.assertEquals("Checkpoint window count ", (int)attrs.get(OperatorContext.CHECKPOINT_WINDOW_COUNT), 120);
  Assert.assertEquals("Operator is stateless ", attrs.get(OperatorContext.STATELESS), true);
  Assert.assertEquals("SPIN MILLIS is set to 20 ", (int)attrs.get(OperatorContext.SPIN_MILLIS), 20);

}
Example 5
Source File: OperatorDiscoveryTest.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
@Test
  public void testLogicalPlanConfiguration() throws Exception
  {
    TestOperator<String, Map<String, Number>> bean = new InputTestOperator<>();
    bean.map.put("key1", new Structured());
    bean.stringArray = new String[]{"one", "two", "three"};
    bean.stringList = Lists.newArrayList("four", "five");
    bean.props = new Properties();
    bean.props.setProperty("key1", "value1");
    bean.structuredArray = new Structured[]{new Structured()};
    bean.genericArray = new String[]{"s1"};
    bean.structuredArray[0].name = "s1";
    bean.color = Color.BLUE;
    bean.booleanProp = true;
    bean.realName = "abc";

    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueSerializer();
    String s = mapper.writeValueAsString(bean);
//    LOG.debug(new JSONObject(s).toString(2));
    //
    Assert.assertTrue("Shouldn't contain field 'realName' !", !s.contains("realName"));
    Assert.assertTrue("Should contain property 'alias' !", s.contains("alias"));
    Assert.assertTrue("Shouldn't contain property 'getterOnly' !", !s.contains("getterOnly"));
    JSONObject jsonObj = new JSONObject(s);

    // create the json dag representation
    JSONObject jsonPlan = new JSONObject();
    jsonPlan.put("streams", new JSONArray());
    JSONObject jsonOper = new JSONObject();
    jsonOper.put("name", "Test Operator");
    jsonOper.put("class", InputTestOperator.class.getName());
    jsonOper.put("properties", jsonObj);
    jsonPlan.put("operators", new JSONArray(Lists.newArrayList(jsonOper)));

    Configuration conf = new Configuration(false);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    // create logical plan from the json
    LogicalPlan lp = lpc.createFromJson(jsonPlan, "jsontest");
    OperatorMeta om = lp.getOperatorMeta("Test Operator");
    Assert.assertTrue(om.getOperator() instanceof InputTestOperator);
    @SuppressWarnings("rawtypes")
    TestOperator beanBack = (InputTestOperator)om.getOperator();

    // The operator deserialized back from json should be same as original operator
    Assert.assertEquals(bean.map, beanBack.map);
    Assert.assertArrayEquals(bean.stringArray, beanBack.stringArray);
    Assert.assertEquals(bean.stringList, beanBack.stringList);
    Assert.assertEquals(bean.props, beanBack.props);
    Assert.assertArrayEquals(bean.structuredArray, beanBack.structuredArray);
    Assert.assertArrayEquals(bean.genericArray, beanBack.genericArray);
    Assert.assertEquals(bean.color, beanBack.color);
    Assert.assertEquals(bean.booleanProp, beanBack.booleanProp);
    Assert.assertEquals(bean.realName, beanBack.realName);
    Assert.assertEquals(bean.getterOnly, beanBack.getterOnly);

  }