org.dmg.pmml.OpType Java Examples

@Override
public MiningModel encodeMiningModel(List<Tree> trees, Integer numIteration, Schema schema){
	Schema segmentSchema = schema.toAnonymousRegressorSchema(DataType.DOUBLE);

	List<MiningModel> miningModels = new ArrayList<>();

	CategoricalLabel categoricalLabel = (CategoricalLabel)schema.getLabel();

	for(int i = 0, rows = categoricalLabel.size(), columns = (trees.size() / rows); i < rows; i++){
		MiningModel miningModel = createMiningModel(FortranMatrixUtil.getRow(trees, rows, columns, i), numIteration, segmentSchema)
			.setOutput(ModelUtil.createPredictedOutput(FieldName.create("lgbmValue(" + categoricalLabel.getValue(i) + ")"), OpType.CONTINUOUS, DataType.DOUBLE));

		miningModels.add(miningModel);
	}

	return MiningModelUtil.createClassification(miningModels, RegressionModel.NormalizationMethod.SOFTMAX, true, schema);
}
 

@Override
public MiningModel encodeMiningModel(List<RegTree> trees, List<Float> weights, float base_score, Integer ntreeLimit, Schema schema){
	Schema segmentSchema = schema.toAnonymousRegressorSchema(DataType.FLOAT);

	List<MiningModel> miningModels = new ArrayList<>();

	CategoricalLabel categoricalLabel = (CategoricalLabel)schema.getLabel();

	for(int i = 0, columns = categoricalLabel.size(), rows = (trees.size() / columns); i < columns; i++){
		MiningModel miningModel = createMiningModel(CMatrixUtil.getColumn(trees, rows, columns, i), (weights != null) ? CMatrixUtil.getColumn(weights, rows, columns, i) : null, base_score, ntreeLimit, segmentSchema)
			.setOutput(ModelUtil.createPredictedOutput(FieldName.create("xgbValue(" + categoricalLabel.getValue(i) + ")"), OpType.CONTINUOUS, DataType.FLOAT));

		miningModels.add(miningModel);
	}

	return MiningModelUtil.createClassification(miningModels, RegressionModel.NormalizationMethod.SOFTMAX, true, schema);
}
 

@Override
public Label encodeLabel(FieldName targetField, List<?> targetCategories, PMMLEncoder encoder){
	DataField dataField;

	if(targetCategories == null){
		targetCategories = LabelUtil.createTargetCategories(this.num_class);

		dataField = encoder.createDataField(targetField, OpType.CATEGORICAL, DataType.INTEGER, targetCategories);
	} else

	{
		if(targetCategories.size() != this.num_class){
			throw new IllegalArgumentException("Expected " + this.num_class + " target categories, got " + targetCategories.size() + " target categories");
		}

		dataField = encoder.createDataField(targetField, OpType.CATEGORICAL, DataType.STRING, targetCategories);
	}

	return new CategoricalLabel(dataField);
}
 

static
public FieldValue create(DataType dataType, Object value){

	if(value instanceof Collection){
		return new CollectionValue(dataType, OpType.CATEGORICAL, (Collection<?>)value);
	}

	switch(dataType){
		case STRING:
			return new CategoricalString(value);
		case BOOLEAN:
			return new CategoricalBoolean(value);
		default:
			return new CategoricalValue(dataType, value);
	}
}
 

@Override
public MiningModel encodeModel(Schema schema){
	LinearSVCModel model = getTransformer();

	Transformation transformation = new AbstractTransformation(){

		@Override
		public Expression createExpression(FieldRef fieldRef){
			return PMMLUtil.createApply(PMMLFunctions.THRESHOLD)
				.addExpressions(fieldRef, PMMLUtil.createConstant(model.getThreshold()));
		}
	};

	Schema segmentSchema = schema.toAnonymousRegressorSchema(DataType.DOUBLE);

	Model linearModel = LinearModelUtil.createRegression(this, model.coefficients(), model.intercept(), segmentSchema)
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("margin"), OpType.CONTINUOUS, DataType.DOUBLE, transformation));

	return MiningModelUtil.createBinaryLogisticClassification(linearModel, 1d, 0d, RegressionModel.NormalizationMethod.NONE, false, schema);
}
 

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	RegexTokenizer transformer = getTransformer();

	if(!transformer.getGaps()){
		throw new IllegalArgumentException("Expected splitter mode, got token matching mode");
	} // End if

	if(transformer.getMinTokenLength() != 1){
		throw new IllegalArgumentException("Expected 1 as minimum token length, got " + transformer.getMinTokenLength() + " as minimum token length");
	}

	Feature feature = encoder.getOnlyFeature(transformer.getInputCol());

	Field<?> field = feature.getField();

	if(transformer.getToLowercase()){
		Apply apply = PMMLUtil.createApply(PMMLFunctions.LOWERCASE, feature.ref());

		field = encoder.createDerivedField(FeatureUtil.createName("lowercase", feature), OpType.CATEGORICAL, DataType.STRING, apply);
	}

	return Collections.singletonList(new DocumentFeature(encoder, field, transformer.getPattern()));
}
 

static
public FieldValue mapValue(MapValues mapValues, Map<String, FieldValue> values){
	String outputColumn = mapValues.getOutputColumn();
	if(outputColumn == null){
		throw new MissingAttributeException(mapValues, PMMLAttributes.MAPVALUES_OUTPUTCOLUMN);
	}

	DataType dataType = mapValues.getDataType(DataType.STRING);

	InlineTable inlineTable = InlineTableUtil.getInlineTable(mapValues);
	if(inlineTable != null){
		Map<String, Object> row = match(inlineTable, values);

		if(row != null){
			Object result = row.get(outputColumn);

			if(result == null){
				throw new InvalidElementException(inlineTable);
			}

			return FieldValueUtil.create(dataType, OpType.CATEGORICAL, result);
		}
	}

	return FieldValueUtil.create(dataType, OpType.CATEGORICAL, mapValues.getDefaultValue());
}
 

static
public Feature encodeIndicatorFeature(Feature feature, Object missingValue, SkLearnEncoder encoder){
	Expression expression = feature.ref();

	if(missingValue != null){
		expression = PMMLUtil.createApply(PMMLFunctions.EQUAL, expression, PMMLUtil.createConstant(missingValue, feature.getDataType()));
	} else

	{
		expression = PMMLUtil.createApply(PMMLFunctions.ISMISSING, expression);
	}

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("missing_indicator", feature), OpType.CATEGORICAL, DataType.BOOLEAN, expression);

	return new BooleanFeature(encoder, derivedField);
}
 

@Test
public void testBuildCategoricalEncoding() {
  List<DataField> dataFields = new ArrayList<>();
  dataFields.add(new DataField(FieldName.create("foo"), OpType.CONTINUOUS, DataType.DOUBLE));
  DataField barField =
      new DataField(FieldName.create("bar"), OpType.CATEGORICAL, DataType.STRING);
  barField.addValues(new Value("b"), new Value("a"));
  dataFields.add(barField);
  DataDictionary dictionary = new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  CategoricalValueEncodings encodings = AppPMMLUtils.buildCategoricalValueEncodings(dictionary);
  assertEquals(2, encodings.getValueCount(1));
  assertEquals(0, encodings.getValueEncodingMap(1).get("b").intValue());
  assertEquals(1, encodings.getValueEncodingMap(1).get("a").intValue());
  assertEquals("b", encodings.getEncodingValueMap(1).get(0));
  assertEquals("a", encodings.getEncodingValueMap(1).get(1));
  assertEquals(Collections.singletonMap(1, 2), encodings.getCategoryCounts());
}
 

public DataField updateDataField(DataField dataField, OpType opType, DataType dataType, SkLearnEncoder encoder){
	FieldName name = dataField.getName();

	if(encoder.isFrozen(name)){
		return dataField;
	}

	switch(dataType){
		case DOUBLE:
			// If the DataField element already specifies a non-default data type, then keep it
			if(!(DataType.DOUBLE).equals(dataField.getDataType())){
				dataType = dataField.getDataType();
			}
			break;
	}

	dataField
		.setOpType(opType)
		.setDataType(dataType);

	return dataField;
}
 

@Override
public OpType getOpType(){

	if(hasTransformers()){
		List<? extends Transformer> transformers = getTransformers();

		for(Transformer transformer : transformers){
			return transformer.getOpType();
		}
	} // End if

	if(hasFinalEstimator()){
		Estimator estimator = getFinalEstimator();

		return estimator.getOpType();
	}

	throw new UnsupportedOperationException();
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	UFunc func = getFunc();

	if(func == null){
		return features;
	}

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		ContinuousFeature continuousFeature = (features.get(i)).toContinuousFeature();

		DerivedField derivedField = encoder.ensureDerivedField(FeatureUtil.createName(func.getName(), continuousFeature), OpType.CONTINUOUS, DataType.DOUBLE, () -> UFuncUtil.encodeUFunc(func, Collections.singletonList(continuousFeature.ref())));

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public Model encodeModel(Schema schema){
	Classifier gbdt = getGBDT();
	MultiOneHotEncoder ohe = getOHE();
	LinearClassifier lr = getLR();

	CategoricalLabel categoricalLabel = (CategoricalLabel)schema.getLabel();

	SchemaUtil.checkSize(2, categoricalLabel);

	List<? extends Number> coef = lr.getCoef();
	List<? extends Number> intercept = lr.getIntercept();

	Schema segmentSchema = schema.toAnonymousSchema();

	MiningModel miningModel = GBDTUtil.encodeModel(gbdt, ohe, coef, Iterables.getOnlyElement(intercept), segmentSchema)
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("decisionFunction"), OpType.CONTINUOUS, DataType.DOUBLE));

	return MiningModelUtil.createBinaryLogisticClassification(miningModel, 1d, 0d, RegressionModel.NormalizationMethod.LOGIT, lr.hasProbabilityDistribution(), schema);
}
 

@Override
public Model encodeModel(Schema schema){
	RGenericVector ada = getObject();

	RGenericVector model = ada.getGenericElement("model");

	RGenericVector trees = model.getGenericElement("trees");
	RDoubleVector alpha = model.getDoubleElement("alpha");

	List<TreeModel> treeModels = encodeTreeModels(trees);

	MiningModel miningModel = new MiningModel(MiningFunction.REGRESSION, ModelUtil.createMiningSchema(null))
		.setSegmentation(MiningModelUtil.createSegmentation(Segmentation.MultipleModelMethod.WEIGHTED_SUM, treeModels, alpha.getValues()))
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("adaValue"), OpType.CONTINUOUS, DataType.DOUBLE));

	return MiningModelUtil.createBinaryLogisticClassification(miningModel, 2d, 0d, RegressionModel.NormalizationMethod.LOGIT, true, schema);
}
 

@Override
public DataField updateDataField(DataField dataField, OpType opType, DataType dataType, SkLearnEncoder encoder){
	FieldName name = dataField.getName();

	if(encoder.isFrozen(name)){
		throw new IllegalArgumentException("Field " + name.getValue() + " is frozen for type information updates");
	}

	dataField
		.setDataType(dataType)
		.setOpType(opType);

	encoder.setDomain(name, this);

	return dataField;
}
 

@Override
public FieldValue evaluate(FieldValue first, FieldValue second){

	// "If one of the input fields of a simple arithmetic function is a missing value, then the result evaluates to missing value"
	if(FieldValueUtil.isMissing(first) || FieldValueUtil.isMissing(second)){
		return FieldValues.MISSING_VALUE;
	}

	DataType dataType = TypeUtil.getCommonDataType(first.getDataType(), second.getDataType());

	Number result;

	try {
		result = evaluate(first.asNumber(), second.asNumber());
	} catch(ArithmeticException ae){
		throw new UndefinedResultException()
			.initCause(ae);
	}

	return FieldValueUtil.create(dataType, OpType.CONTINUOUS, result);
}
 

static
public OpType getOpType(DataType dataType){

	switch(dataType){
		case STRING:
			return OpType.CATEGORICAL;
		case INTEGER:
		case FLOAT:
		case DOUBLE:
			return OpType.CONTINUOUS;
		case BOOLEAN:
			return OpType.CATEGORICAL;
		case DATE:
		case DATE_TIME:
			return OpType.ORDINAL;
		default:
			throw new IllegalArgumentException();
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String pattern = getPattern();
	String replacement = getReplacement();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.REPLACE)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(pattern, DataType.STRING), PMMLUtil.createConstant(replacement, DataType.STRING));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("replace", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

@Override
public Label encodeLabel(FieldName targetField, List<?> targetCategories, PMMLEncoder encoder){
	DataField dataField;

	if(targetCategories == null){
		targetCategories = LabelUtil.createTargetCategories(this.num_class_);

		dataField = encoder.createDataField(targetField, OpType.CATEGORICAL, DataType.INTEGER, targetCategories);
	} else

	{
		if(targetCategories.size() != this.num_class_){
			throw new IllegalArgumentException("Expected " + this.num_class_ + " target categories, got " + targetCategories.size() + " target categories");
		}

		dataField = encoder.createDataField(targetField, OpType.CATEGORICAL, DataType.STRING, targetCategories);
	}

	return new CategoricalLabel(dataField);
}
 

public static DataDictionary buildDataDictionary(
    InputSchema schema,
    CategoricalValueEncodings categoricalValueEncodings) {
  List<String> featureNames = schema.getFeatureNames();

  List<DataField> dataFields = new ArrayList<>();
  for (int featureIndex = 0; featureIndex < featureNames.size(); featureIndex++) {
    String featureName = featureNames.get(featureIndex);
    OpType opType;
    DataType dataType;
    if (schema.isNumeric(featureName)) {
      opType = OpType.CONTINUOUS;
      dataType = DataType.DOUBLE;
    } else if (schema.isCategorical(featureName)) {
      opType = OpType.CATEGORICAL;
      dataType = DataType.STRING;
    } else {
      // Don't know
      opType = null;
      dataType = null;
    }
    DataField field = new DataField(FieldName.create(featureName), opType, dataType);
    if (schema.isCategorical(featureName)) {
      Objects.requireNonNull(categoricalValueEncodings);
      categoricalValueEncodings.getEncodingValueMap(featureIndex).entrySet().stream().
          sorted(Comparator.comparing(Map.Entry::getKey)).
          map(Map.Entry::getValue).
          forEach(value -> field.addValues(new Value(value)));
    }
    dataFields.add(field);
  }

  return new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
}
 

@Override
public DataField createDataField(FieldName name, OpType opType, DataType dataType, List<?> values){

	if(dataType == null){
		dataType = TypeUtil.getDataType(values);
	}

	return super.createDataField(name, opType, dataType, values);
}
 

private static void checkDataField(DataField field, String name, Boolean categorical) {
  assertEquals(name, field.getName().getValue());
  if (categorical == null) {
    assertNull(field.getOpType());
    assertNull(field.getDataType());
  } else if (categorical) {
    assertEquals(OpType.CATEGORICAL, field.getOpType());
    assertEquals(DataType.STRING, field.getDataType());
  } else {
    assertEquals(OpType.CONTINUOUS, field.getOpType());
    assertEquals(DataType.DOUBLE, field.getDataType());
  }
}
 

static
public FieldValue evaluateDiscretize(Discretize discretize, EvaluationContext context){
	FieldValue value = context.evaluate(ensureField(discretize));

	if(FieldValueUtil.isMissing(value)){
		return FieldValueUtil.create(discretize.getDataType(DataType.STRING), OpType.CATEGORICAL, discretize.getMapMissingTo());
	}

	return DiscretizationUtil.discretize(discretize, value);
}
 

@Override
public MiningModel encodeMiningModel(List<Tree> trees, Integer numIteration, Schema schema){
	Schema segmentSchema = schema.toAnonymousRegressorSchema(DataType.DOUBLE);

	MiningModel miningModel = createMiningModel(trees, numIteration, segmentSchema)
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("lgbmValue"), OpType.CONTINUOUS, DataType.DOUBLE));

	return MiningModelUtil.createBinaryLogisticClassification(miningModel, BinomialLogisticRegression.this.sigmoid_, 0d, RegressionModel.NormalizationMethod.LOGIT, true, schema);
}
 

@Override
public MiningModel encodeMiningModel(List<RegTree> trees, List<Float> weights, float base_score, Integer ntreeLimit, Schema schema){
	Schema segmentSchema = schema.toAnonymousSchema();

	MiningModel miningModel = createMiningModel(trees, weights, base_score, ntreeLimit, segmentSchema)
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("xgbValue"), OpType.CONTINUOUS, DataType.FLOAT));

	return MiningModelUtil.createRegression(miningModel, RegressionModel.NormalizationMethod.LOGIT, schema);
}
 

@Override
public MiningModel encodeMiningModel(List<RegTree> trees, List<Float> weights, float base_score, Integer ntreeLimit, Schema schema){
	Schema segmentSchema = schema.toAnonymousSchema();

	MiningModel miningModel = createMiningModel(trees, weights, base_score, ntreeLimit, segmentSchema)
		.setOutput(ModelUtil.createPredictedOutput(FieldName.create("xgbValue"), OpType.CONTINUOUS, DataType.FLOAT));

	return MiningModelUtil.createRegression(miningModel, RegressionModel.NormalizationMethod.EXP, schema);
}
 

@Override
public FieldValue evaluate(List<FieldValue> arguments){
	StorelessUnivariateStatistic statistic = createStatistic();

	DataType dataType = null;

	for(int i = 0; i < arguments.size(); i++){
		FieldValue value = getOptionalArgument(arguments, i);

		// "Missing values in the input to an aggregate function are simply ignored"
		if(FieldValueUtil.isMissing(value)){
			continue;
		}

		statistic.increment((value.asNumber()).doubleValue());

		if(dataType != null){
			dataType = TypeUtil.getCommonDataType(dataType, value.getDataType());
		} else

		{
			dataType = value.getDataType();
		}
	}

	// "If all inputs are missing, then the result evaluates to a missing value"
	if(statistic.getN() == 0){
		return FieldValues.MISSING_VALUE;
	}

	Double result = statistic.getResult();

	return FieldValueUtil.create(getResultDataType(dataType), OpType.CONTINUOUS, result);
}
 

@Test
public void continuousIntegerList(){
	FieldValue list = FieldValueUtil.create(TypeInfos.CONTINUOUS_INTEGER, Arrays.asList(1, 2, 3));

	assertEquals(DataType.INTEGER, list.getDataType());
	assertEquals(OpType.CONTINUOUS, list.getOpType());
}
 

@Override
public Feature encode(WildcardFeature wildcardFeature, List<?> values){
	PMMLEncoder encoder = wildcardFeature.getEncoder();

	if(values == null || values.isEmpty()){
		DataField dataField = (DataField)encoder.getField(wildcardFeature.getName());

		dataField.setOpType(OpType.CATEGORICAL);

		return new ObjectFeature(encoder, dataField.getName(), dataField.getDataType());
	}

	return wildcardFeature.toCategoricalFeature(standardizeValues(wildcardFeature.getDataType(), values));
}
 

@Test
public void ordinalInteger(){
	OrdinalValue zero = (OrdinalValue)FieldValueUtil.create(DataType.INTEGER, OpType.ORDINAL, 0);
	OrdinalValue one = (OrdinalValue)FieldValueUtil.create(DataType.INTEGER, OpType.ORDINAL, 1);

	assertTrue(zero.equalsValue("0"));
	assertTrue(zero.equalsValue("0.0"));

	assertTrue(zero.compareToValue("-1") > 0);
	assertTrue(zero.compareToValue("0") == 0);
	assertTrue(zero.compareToValue("1") < 0);

	assertTrue(zero.compareTo(zero) == 0);
	assertTrue(zero.compareTo(one) < 0);

	assertTrue(one.compareTo(zero) > 0);
	assertTrue(one.compareTo(one) == 0);

	TypeInfo typeInfo = new SimpleTypeInfo(DataType.INTEGER, OpType.ORDINAL, Arrays.asList(1, 0));

	zero = (OrdinalValue)FieldValueUtil.create(typeInfo, zero.getValue());

	assertTrue(zero.compareTo(zero) == 0);
	assertTrue(zero.compareTo(one) > 0);

	one = (OrdinalValue)FieldValueUtil.create(typeInfo, one.getValue());

	assertTrue(one.compareTo(zero) < 0);
	assertTrue(one.compareTo(one) == 0);
}