org.dmg.pmml.tree.ComplexNode Java Examples

@Test
public void intern(){
	ScoreDistribution left = new ScoreDistribution("event", 0.33d);
	ScoreDistribution right = new ScoreDistribution("event", 0.33d);

	Node leftChild = createNode(left);
	Node rightChild = createNode(right);

	Node root = new ComplexNode(True.INSTANCE)
		.addNodes(leftChild, rightChild);

	TreeModel treeModel = new TreeModel()
		.setNode(root);

	for(int i = 0; i < 2; i++){
		assertNotSame((leftChild.getScoreDistributions()).get(i), (rightChild.getScoreDistributions()).get(i));
	}

	ScoreDistributionInterner interner = new ScoreDistributionInterner();
	interner.applyTo(treeModel);

	for(int i = 0; i < 2; i++){
		assertSame((leftChild.getScoreDistributions()).get(i), (rightChild.getScoreDistributions()).get(i));
	}
}
 

static
private Node createNode(ScoreDistribution event){
	ScoreDistribution noEvent = new ScoreDistribution("no-event", 1d - NumberUtil.asDouble(event.getRecordCount()));

	Node node = new ComplexNode()
		.addScoreDistributions(event, noEvent);

	return node;
}
 

private static PMML buildDummyClassificationModel(int numTrees) {
  PMML pmml = PMMLUtils.buildSkeletonPMML();

  List<DataField> dataFields = new ArrayList<>();
  DataField predictor =
      new DataField(FieldName.create("color"), OpType.CATEGORICAL, DataType.STRING);
  predictor.addValues(new Value("yellow"), new Value("red"));
  dataFields.add(predictor);
  DataField target =
      new DataField(FieldName.create("fruit"), OpType.CATEGORICAL, DataType.STRING);
  target.addValues(new Value("banana"), new Value("apple"));
  dataFields.add(target);
  DataDictionary dataDictionary =
      new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  pmml.setDataDictionary(dataDictionary);

  List<MiningField> miningFields = new ArrayList<>();
  MiningField predictorMF = new MiningField(FieldName.create("color"))
      .setOpType(OpType.CATEGORICAL)
      .setUsageType(MiningField.UsageType.ACTIVE)
      .setImportance(0.5);
  miningFields.add(predictorMF);
  MiningField targetMF = new MiningField(FieldName.create("fruit"))
      .setOpType(OpType.CATEGORICAL)
      .setUsageType(MiningField.UsageType.PREDICTED);
  miningFields.add(targetMF);
  MiningSchema miningSchema = new MiningSchema(miningFields);

  double dummyCount = 2.0;
  Node rootNode =
    new ComplexNode().setId("r").setRecordCount(dummyCount).setPredicate(new True());

  double halfCount = dummyCount / 2;

  Node left = new ComplexNode().setId("r-").setRecordCount(halfCount).setPredicate(new True());
  left.addScoreDistributions(new ScoreDistribution("apple", halfCount));
  Node right = new ComplexNode().setId("r+").setRecordCount(halfCount)
      .setPredicate(new SimpleSetPredicate(FieldName.create("color"),
                                           SimpleSetPredicate.BooleanOperator.IS_NOT_IN,
                                           new Array(Array.Type.STRING, "red")));
  right.addScoreDistributions(new ScoreDistribution("banana", halfCount));

  rootNode.addNodes(right, left);

  TreeModel treeModel = new TreeModel(MiningFunction.CLASSIFICATION, miningSchema, rootNode)
      .setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT)
      .setMissingValueStrategy(TreeModel.MissingValueStrategy.DEFAULT_CHILD);

  if (numTrees > 1) {
    MiningModel miningModel = new MiningModel(MiningFunction.CLASSIFICATION, miningSchema);
    List<Segment> segments = new ArrayList<>();
    for (int i = 0; i < numTrees; i++) {
      segments.add(new Segment()
          .setId(Integer.toString(i))
          .setPredicate(new True())
          .setModel(treeModel)
          .setWeight(1.0));
    }
    miningModel.setSegmentation(
        new Segmentation(Segmentation.MultipleModelMethod.WEIGHTED_MAJORITY_VOTE, segments));
    pmml.addModels(miningModel);
  } else {
    pmml.addModels(treeModel);
  }

  return pmml;
}
 

public static PMML buildDummyRegressionModel() {
  PMML pmml = PMMLUtils.buildSkeletonPMML();

  List<DataField> dataFields = new ArrayList<>();
  dataFields.add(new DataField(FieldName.create("foo"), OpType.CONTINUOUS, DataType.DOUBLE));
  dataFields.add(new DataField(FieldName.create("bar"), OpType.CONTINUOUS, DataType.DOUBLE));
  DataDictionary dataDictionary =
      new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  pmml.setDataDictionary(dataDictionary);

  List<MiningField> miningFields = new ArrayList<>();
  MiningField predictorMF = new MiningField(FieldName.create("foo"))
      .setOpType(OpType.CONTINUOUS)
      .setUsageType(MiningField.UsageType.ACTIVE)
      .setImportance(0.5);
  miningFields.add(predictorMF);
  MiningField targetMF = new MiningField(FieldName.create("bar"))
      .setOpType(OpType.CONTINUOUS)
      .setUsageType(MiningField.UsageType.PREDICTED);
  miningFields.add(targetMF);
  MiningSchema miningSchema = new MiningSchema(miningFields);

  double dummyCount = 2.0;
  Node rootNode =
      new ComplexNode().setId("r").setRecordCount(dummyCount).setPredicate(new True());

  double halfCount = dummyCount / 2;

  Node left = new ComplexNode()
      .setId("r-")
      .setRecordCount(halfCount)
      .setPredicate(new True())
      .setScore("-2.0");
  Node right = new ComplexNode().setId("r+").setRecordCount(halfCount)
      .setPredicate(new SimplePredicate(FieldName.create("foo"),
                                        SimplePredicate.Operator.GREATER_THAN,
                                        "3.14"))
      .setScore("2.0");

  rootNode.addNodes(right, left);

  TreeModel treeModel = new TreeModel(MiningFunction.REGRESSION, miningSchema, rootNode)
      .setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT)
      .setMissingValueStrategy(TreeModel.MissingValueStrategy.DEFAULT_CHILD)
      .setMiningSchema(miningSchema);

  pmml.addModels(treeModel);

  return pmml;
}
 

private TreeModel toTreeModel(DecisionTreeModel dtModel,
                              CategoricalValueEncodings categoricalValueEncodings,
                              IntLongMap nodeIDCounts) {

  boolean classificationTask = dtModel.algo().equals(Algo.Classification());
  Preconditions.checkState(classificationTask == inputSchema.isClassification());

  Node root = new ComplexNode();
  root.setId("r");

  Queue<Node> modelNodes = new ArrayDeque<>();
  modelNodes.add(root);

  Queue<Pair<org.apache.spark.mllib.tree.model.Node,Split>> treeNodes = new ArrayDeque<>();
  treeNodes.add(new Pair<>(dtModel.topNode(), null));

  while (!treeNodes.isEmpty()) {

    Pair<org.apache.spark.mllib.tree.model.Node,Split> treeNodePredicate = treeNodes.remove();
    Node modelNode = modelNodes.remove();

    // This is the decision that got us here from the parent, if any;
    // not the predicate at this node
    Predicate predicate = buildPredicate(treeNodePredicate.getSecond(),
                                         categoricalValueEncodings);
    modelNode.setPredicate(predicate);

    org.apache.spark.mllib.tree.model.Node treeNode = treeNodePredicate.getFirst();
    long nodeCount = nodeIDCounts.get(treeNode.id());
    modelNode.setRecordCount((double) nodeCount);

    if (treeNode.isLeaf()) {

      Predict prediction = treeNode.predict();
      int targetEncodedValue = (int) prediction.predict();
      if (classificationTask) {
        Map<Integer,String> targetEncodingToValue =
            categoricalValueEncodings.getEncodingValueMap(inputSchema.getTargetFeatureIndex());
        double predictedProbability = prediction.prob();
        Preconditions.checkState(predictedProbability >= 0.0 && predictedProbability <= 1.0);
        // Not sure how nodeCount == 0 can happen but it does in the MLlib model
        long effectiveNodeCount = Math.max(1, nodeCount);
        // Problem: MLlib only gives a predicted class and its probability, and no distribution
        // over the rest. Infer that the rest of the probability is evenly distributed.
        double restProbability = (1.0 - predictedProbability) / (targetEncodingToValue.size() - 1);

        targetEncodingToValue.forEach((encodedValue, value) -> {
          double probability = encodedValue == targetEncodedValue ? predictedProbability : restProbability;
          // Yes, recordCount may be fractional; it's a relative indicator
          double recordCount = probability * effectiveNodeCount;
          if (recordCount > 0.0) {
            ScoreDistribution distribution = new ScoreDistribution(value, recordCount);
            // Not "confident" enough in the "probability" to call it one
            distribution.setConfidence(probability);
            modelNode.addScoreDistributions(distribution);
          }
        });
      } else {
        modelNode.setScore(Double.toString(targetEncodedValue));
      }

    } else {

      Split split = treeNode.split().get();

      Node positiveModelNode = new ComplexNode().setId(modelNode.getId() + "+");
      Node negativeModelNode = new ComplexNode().setId(modelNode.getId() + "-");
      modelNode.addNodes(positiveModelNode, negativeModelNode);

      org.apache.spark.mllib.tree.model.Node rightTreeNode = treeNode.rightNode().get();
      org.apache.spark.mllib.tree.model.Node leftTreeNode = treeNode.leftNode().get();

      boolean defaultRight = nodeIDCounts.get(rightTreeNode.id()) > nodeIDCounts.get(leftTreeNode.id());
      modelNode.setDefaultChild(defaultRight ? positiveModelNode.getId() : negativeModelNode.getId());

      // Right node is "positive", so carries the predicate. It must evaluate first
      // and therefore come first in the tree
      modelNodes.add(positiveModelNode);
      modelNodes.add(negativeModelNode);
      treeNodes.add(new Pair<>(rightTreeNode, split));
      treeNodes.add(new Pair<>(leftTreeNode, null));

    }

  }

  return new TreeModel()
      .setNode(root)
      .setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT)
      .setMissingValueStrategy(TreeModel.MissingValueStrategy.DEFAULT_CHILD);
}
 

@Override
public VisitorAction visit(ComplexNode complexNode){
	return super.visit(complexNode);
}
 

@Override
public Node unmarshal(ComplexNode value){
	NodeTransformer nodeTransformer = NodeAdapter.NODE_TRANSFORMER_PROVIDER.get();

	return nodeTransformer.fromComplexNode(value);
}
 

@Override
public ComplexNode marshal(Node node){
	NodeTransformer nodeTransformer = NodeAdapter.NODE_TRANSFORMER_PROVIDER.get();

	return nodeTransformer.toComplexNode(node);
}
 

@Test
public void jsonClone() throws Exception {
	DataField dataField = new DataField(FieldName.create("x"), OpType.CATEGORICAL, DataType.BOOLEAN);

	DataDictionary dataDictionary = new DataDictionary()
		.addDataFields(dataField);

	MiningField miningField = new MiningField(FieldName.create("x"));

	MiningSchema miningSchema = new MiningSchema()
		.addMiningFields(miningField);

	assertSame(dataField.getName(), miningField.getName());

	SimplePredicate simplePredicate = new SimplePredicate(FieldName.create("x"), SimplePredicate.Operator.IS_NOT_MISSING, null);

	Node node = new ComplexNode(simplePredicate);

	TreeModel treeModel = new TreeModel()
		.setMiningSchema(miningSchema)
		.setNode(node);

	PMML pmml = new PMML()
		.setDataDictionary(dataDictionary)
		.addModels(treeModel);

	DirectByteArrayOutputStream buffer = new DirectByteArrayOutputStream(1024);

	JacksonUtil.writePMML(pmml, buffer);

	PMML jsonPmml;

	try(InputStream is = buffer.getInputStream()){
		jsonPmml = JacksonUtil.readPMML(is);
	}

	DataDictionary jsonDataDictionary = jsonPmml.getDataDictionary();

	List<DataField> jsonDataFields = jsonDataDictionary.getDataFields();

	assertEquals(1, jsonDataFields.size());

	DataField jsonDataField = jsonDataFields.get(0);

	assertEquals(dataField.getName(), jsonDataField.getName());
	assertEquals(dataField.getOpType(), jsonDataField.getOpType());
	assertEquals(dataField.getDataType(), jsonDataField.getDataType());

	List<Model> jsonModels = jsonPmml.getModels();

	assertEquals(1, jsonModels.size());

	TreeModel jsonTreeModel = (TreeModel)jsonModels.get(0);

	MiningSchema jsonMiningSchema = jsonTreeModel.getMiningSchema();

	List<MiningField> jsonMiningFields = jsonMiningSchema.getMiningFields();

	assertEquals(1, jsonMiningFields.size());

	MiningField jsonMiningField = jsonMiningFields.get(0);

	assertEquals(miningField.getName(), jsonMiningField.getName());
	assertEquals(miningField.getUsageType(), jsonMiningField.getUsageType());

	assertSame(jsonDataField.getName(), jsonMiningField.getName());

	Node jsonNode = jsonTreeModel.getNode();

	SimplePredicate jsonSimplePredicate = (SimplePredicate)jsonNode.getPredicate();

	assertEquals(simplePredicate.getField(), jsonSimplePredicate.getField());
	assertEquals(simplePredicate.getOperator(), jsonSimplePredicate.getOperator());

	assertSame(jsonDataField.getName(), jsonSimplePredicate.getField());
	assertSame(jsonMiningField.getName(), jsonSimplePredicate.getField());
}