org.dmg.pmml.DerivedField Java Examples

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String pattern = getPattern();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.MATCHES)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(pattern, DataType.STRING));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("matches", feature), OpType.CATEGORICAL, DataType.BOOLEAN, apply);

	return Collections.singletonList(new BooleanFeature(encoder, derivedField));
}
 

static
private FieldName prepareInputField(FunctionExpression.Argument argument, OpType opType, DataType dataType, RExpEncoder encoder){
	Expression expression = argument.getExpression();

	if(expression instanceof FieldRef){
		FieldRef fieldRef = (FieldRef)expression;

		return fieldRef.getField();
	} else

	if(expression instanceof Apply){
		Apply apply = (Apply)expression;

		DerivedField derivedField = encoder.createDerivedField(FieldName.create((argument.formatExpression()).trim()), opType, dataType, apply);

		return derivedField.getName();
	} else

	{
		throw new IllegalArgumentException();
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Integer begin = getBegin();
	Integer end = getEnd();

	if((begin < 0) || (end < begin)){
		throw new IllegalArgumentException();
	}

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.SUBSTRING)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(begin + 1, DataType.INTEGER), PMMLUtil.createConstant((end - begin), DataType.INTEGER));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("substring", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

static
public FieldValue evaluate(DerivedField derivedField, EvaluationContext context){
	FieldName name = derivedField.getName();
	if(name == null){
		throw new MissingAttributeException(derivedField, PMMLAttributes.DERIVEDFIELD_NAME);
	}

	SymbolTable<FieldName> symbolTable = EvaluationContext.DERIVEDFIELD_GUARD_PROVIDER.get();

	if(symbolTable != null){
		symbolTable.lock(name);
	}

	try {
		return evaluateTypedExpressionContainer(derivedField, context);
	} finally {

		if(symbolTable != null){
			symbolTable.release(name);
		}
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String pattern = getPattern();
	String replacement = getReplacement();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.REPLACE)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(pattern, DataType.STRING), PMMLUtil.createConstant(replacement, DataType.STRING));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("replace", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

static
public Feature encodeIndicatorFeature(Feature feature, Object missingValue, SkLearnEncoder encoder){
	Expression expression = feature.ref();

	if(missingValue != null){
		expression = PMMLUtil.createApply(PMMLFunctions.EQUAL, expression, PMMLUtil.createConstant(missingValue, feature.getDataType()));
	} else

	{
		expression = PMMLUtil.createApply(PMMLFunctions.ISMISSING, expression);
	}

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("missing_indicator", feature), OpType.CATEGORICAL, DataType.BOOLEAN, expression);

	return new BooleanFeature(encoder, derivedField);
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	UFunc func = getFunc();

	if(func == null){
		return features;
	}

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		ContinuousFeature continuousFeature = (features.get(i)).toContinuousFeature();

		DerivedField derivedField = encoder.ensureDerivedField(FeatureUtil.createName(func.getName(), continuousFeature), OpType.CONTINUOUS, DataType.DOUBLE, () -> UFuncUtil.encodeUFunc(func, Collections.singletonList(continuousFeature.ref())));

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String function = getFunction();

	if(features.size() <= 1){
		return features;
	}

	Apply apply = PMMLUtil.createApply(translateFunction(function));

	for(Feature feature : features){
		apply.addExpressions(feature.ref());
	}

	FieldName name = FeatureUtil.createName(function, features);

	DerivedField derivedField = encoder.createDerivedField(name, OpType.CONTINUOUS, DataType.DOUBLE, apply);

	return Collections.singletonList(new ContinuousFeature(encoder, derivedField));
}
 

static
private Field<?> resolveField(FieldName name, MiningModelEvaluationContext context){

	while(context != null){
		OutputField outputField = context.getOutputField(name);
		if(outputField != null){
			return outputField;
		}

		DerivedField localDerivedField = context.getLocalDerivedField(name);
		if(localDerivedField != null){
			return localDerivedField;
		}

		context = context.getParent();
	}

	return null;
}
 

private void processDerivedFields(HasDerivedFields<?> hasDerivedFields){

		if(hasDerivedFields.hasDerivedFields()){
			List<DerivedField> derivedFields = hasDerivedFields.getDerivedFields();

			for(ListIterator<DerivedField> it = derivedFields.listIterator(); it.hasNext(); ){
				DerivedField derivedField = it.next();

				if(derivedField.hasValues()){
					it.set(new RichDerivedField(derivedField));
				}
			}
		}
	}
 

@Override
public void addDerivedField(DerivedField derivedField){

	try {
		super.addDerivedField(derivedField);
	} catch(RuntimeException re){
		FieldName name = derivedField.getName();

		String message = "Field " + name.getValue() + " is already defined. " +
			"Please refactor the pipeline so that it would not contain duplicate field declarations, " +
			"or use the " + (Alias.class).getName() + " wrapper class to override the default name with a custom name (eg. " + Alias.formatAliasExample() + ")";

		throw new IllegalArgumentException(message, re);
	}
}
 

public void renameFeature(Feature feature, FieldName renamedName){
	FieldName name = feature.getName();

	org.dmg.pmml.Field<?> pmmlField = getField(name);

	if(pmmlField instanceof DataField){
		throw new IllegalArgumentException("User input field " + name.getValue() + " cannot be renamed");
	}

	DerivedField derivedField = removeDerivedField(name);

	try {
		Field field = Feature.class.getDeclaredField("name");

		if(!field.isAccessible()){
			field.setAccessible(true);
		}

		field.set(feature, renamedName);
	} catch(ReflectiveOperationException roe){
		throw new RuntimeException(roe);
	}

	derivedField.setName(renamedName);

	addDerivedField(derivedField);
}
 

public void addField(Field<?> field){
	RExpEncoder encoder = getEncoder();

	Feature feature = new ContinuousFeature(encoder, field);

	if(field instanceof DerivedField){
		DerivedField derivedField = (DerivedField)field;

		Expression expression = derivedField.getExpression();
		if(expression instanceof Apply){
			Apply apply = (Apply)expression;

			if(checkApply(apply, PMMLFunctions.POW, FieldRef.class, Constant.class)){
				List<Expression> expressions = apply.getExpressions();

				FieldRef fieldRef = (FieldRef)expressions.get(0);
				Constant constant = (Constant)expressions.get(1);

				try {
					String string = ValueUtil.asString(constant.getValue());

					int power = Integer.parseInt(string);

					feature = new PowerFeature(encoder, fieldRef.getField(), DataType.DOUBLE, power);
				} catch(NumberFormatException nfe){
					// Ignored
				}
			}
		}
	}

	putFeature(field.getName(), feature);

	this.fields.add(field);
}
 

private Expression getOutputExpression(NeuralOutput neuralOutput){
	DerivedField derivedField = neuralOutput.getDerivedField();
	if(derivedField == null){
		throw new MissingElementException(neuralOutput, PMMLElements.NEURALOUTPUT_DERIVEDFIELD);
	}

	Expression expression = ExpressionUtil.ensureExpression(derivedField);

	if(expression instanceof FieldRef){
		FieldRef fieldRef = (FieldRef)expression;

		FieldName name = fieldRef.getField();
		if(name == null){
			throw new MissingAttributeException(fieldRef, org.dmg.pmml.PMMLAttributes.FIELDREF_FIELD);
		}

		Field<?> field = resolveField(name);
		if(field == null){
			throw new MissingFieldException(name, fieldRef);
		} // End if

		if(field instanceof DataField){
			return expression;
		} else

		if(field instanceof DerivedField){
			DerivedField targetDerivedField = (DerivedField)field;

			Expression targetExpression = ExpressionUtil.ensureExpression(targetDerivedField);

			return targetExpression;
		} else

		{
			throw new InvalidAttributeException(fieldRef, org.dmg.pmml.PMMLAttributes.FIELDREF_FIELD, name);
		}
	}

	return expression;
}
 

@Override
public VisitorAction visit(BayesInputs bayesInputs){

	if(bayesInputs.hasBayesInputs()){
		List<BayesInput> content = bayesInputs.getBayesInputs();

		for(ListIterator<BayesInput> it = content.listIterator(); it.hasNext(); ){
			BayesInput bayesInput = it.next();

			FieldName name = bayesInput.getField();
			if(name == null){
				throw new MissingAttributeException(bayesInput, org.dmg.pmml.naive_bayes.PMMLAttributes.BAYESINPUT_FIELD);
			}

			DataType dataType;

			DerivedField derivedField = bayesInput.getDerivedField();
			if(derivedField != null){
				dataType = derivedField.getDataType();

				if(dataType == null){
					throw new MissingAttributeException(derivedField, PMMLAttributes.DERIVEDFIELD_DATATYPE);
				}
			} else

			{
				dataType = resolveDataType(name);
			} // End if

			if(dataType != null){
				it.set(new RichBayesInput(dataType, bayesInput));
			}
		}
	}

	return super.visit(bayesInputs);
}
 

static
public Feature encodeIndexFeature(Feature feature, List<?> categories, List<? extends Number> indexCategories, Number mapMissingTo, Number defaultValue, DataType dataType, SkLearnEncoder encoder){
	ClassDictUtil.checkSize(categories, indexCategories);

	encoder.toCategorical(feature.getName(), categories);

	Supplier<MapValues> mapValuesSupplier = () -> {
		MapValues mapValues = PMMLUtil.createMapValues(feature.getName(), categories, indexCategories)
			.setMapMissingTo(mapMissingTo)
			.setDefaultValue(defaultValue);

		return mapValues;
	};

	DerivedField derivedField = encoder.ensureDerivedField(FeatureUtil.createName("encoder", feature), OpType.CATEGORICAL, dataType, mapValuesSupplier);

	Feature encodedFeature = new IndexFeature(encoder, derivedField, indexCategories);

	Feature result = new CategoricalFeature(encoder, feature, categories){

		@Override
		public ContinuousFeature toContinuousFeature(){
			return encodedFeature.toContinuousFeature();
		}
	};

	return result;
}
 

private void scaleFeatures(RExpEncoder encoder){
	RGenericVector mvr = getObject();

	RDoubleVector scale = mvr.getDoubleElement("scale", false);
	if(scale == null){
		return;
	}

	List<Feature> features = encoder.getFeatures();

	if(scale.size() != features.size()){
		throw new IllegalArgumentException();
	}

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);
		Double factor = scale.getValue(i);

		if(ValueUtil.isOne(factor)){
			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		Apply apply = PMMLUtil.createApply(PMMLFunctions.DIVIDE, continuousFeature.ref(), PMMLUtil.createConstant(factor));

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("scale", feature), OpType.CONTINUOUS, DataType.DOUBLE, apply);

		features.set(i, new ContinuousFeature(encoder, derivedField));
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Object dtype = getDType();
	String expr = getExpr();

	Scope scope = new DataFrameScope(FieldName.create("X"), features);

	Expression expression = ExpressionTranslator.translate(expr, scope);

	DataType dataType;

	if(dtype != null){
		dataType = TransformerUtil.getDataType(dtype);
	} else

	{
		if(ExpressionTranslator.isString(expression, scope)){
			dataType = DataType.STRING;
		} else

		{
			dataType = DataType.DOUBLE;
		}
	}

	OpType opType = TransformerUtil.getOpType(dataType);

	DerivedField derivedField = encoder.createDerivedField(FieldName.create("eval(" + expr + ")"), opType, dataType, expression);

	return Collections.singletonList(new ContinuousFeature(encoder, derivedField));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String function = getFunction();
	Boolean trimBlanks = getTrimBlanks();

	if(function == null && !trimBlanks){
		return features;
	}

	List<Feature> result = new ArrayList<>();

	for(Feature feature : features){
		Expression expression = feature.ref();

		if(function != null){
			expression = PMMLUtil.createApply(translateFunction(function), expression);
		} // End if

		if(trimBlanks){
			expression = PMMLUtil.createApply(PMMLFunctions.TRIMBLANKS, expression);
		}

		Field<?> field = encoder.toCategorical(feature.getName(), Collections.emptyList());

		// XXX: Should have been set by the previous transformer
		field.setDataType(DataType.STRING);

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("normalize", feature), OpType.CATEGORICAL, DataType.STRING, expression);

		feature = new StringFeature(encoder, derivedField);

		result.add(feature);
	}

	return result;
}
 

private void scaleFeatures(RExpEncoder encoder){
	RGenericVector svm = getObject();

	RDoubleVector sv = svm.getDoubleElement("SV");
	RBooleanVector scaled = svm.getBooleanElement("scaled");
	RGenericVector xScale = svm.getGenericElement("x.scale");

	RStringVector rowNames = sv.dimnames(0);
	RStringVector columnNames = sv.dimnames(1);

	List<Feature> features = encoder.getFeatures();

	if((scaled.size() != columnNames.size()) || (scaled.size() != features.size())){
		throw new IllegalArgumentException();
	}

	RDoubleVector xScaledCenter = xScale.getDoubleElement("scaled:center");
	RDoubleVector xScaledScale = xScale.getDoubleElement("scaled:scale");

	for(int i = 0; i < columnNames.size(); i++){
		String columnName = columnNames.getValue(i);

		if(!scaled.getValue(i)){
			continue;
		}

		Feature feature = features.get(i);

		Double center = xScaledCenter.getElement(columnName);
		Double scale = xScaledScale.getElement(columnName);

		if(ValueUtil.isZero(center) && ValueUtil.isOne(scale)){
			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		Expression expression = continuousFeature.ref();

		if(!ValueUtil.isZero(center)){
			expression = PMMLUtil.createApply(PMMLFunctions.SUBTRACT, expression, PMMLUtil.createConstant(center));
		} // End if

		if(!ValueUtil.isOne(scale)){
			expression = PMMLUtil.createApply(PMMLFunctions.DIVIDE, expression, PMMLUtil.createConstant(scale));
		}

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("scale", feature), OpType.CONTINUOUS, DataType.DOUBLE, expression);

		features.set(i, new ContinuousFeature(encoder, derivedField));
	}
}
 

public CustomCenterFields(List<DerivedField> derivedFields){
	super(derivedFields);
}
 

public DerivedField getDerivedField(FieldName name){
	return this.derivedFields.get(name);
}
 

public DerivedField getLocalDerivedField(FieldName name){
	return this.localDerivedFields.get(name);
}
 

@Override
public Map<FieldName, DerivedField> load(TransformationDictionary transformationDictionary){
	return IndexableUtil.buildMap(transformationDictionary.getDerivedFields());
}
 

@Override
public Map<FieldName, DerivedField> load(LocalTransformations localTransformations){
	return IndexableUtil.buildMap(localTransformations.getDerivedFields());
}
 

public DerivedField getLocalDerivedField(FieldName name){
	MiningModelEvaluator miningModelEvaluator = getModelEvaluator();

	return miningModelEvaluator.getLocalDerivedField(name);
}
 

public RichDerivedField(DerivedField derivedField){
	ReflectionUtil.copyState(derivedField, this);
}
 

private DerivedFieldLoader(FieldName name, String column, DerivedField derivedField, MiningField miningField){
	super(name, column);

	setDerivedField(derivedField);
	setMiningField(miningField);
}
 

@Override
public FieldValue prepare(Object value){
	DerivedField derivedField = getDerivedField();
	MiningField miningField = getMiningField();

	if(miningField != null){
		return InputFieldUtil.prepareInputValue(derivedField, miningField, value);
	}

	TypeInfo typeInfo = new TypeInfo(){

		@Override
		public DataType getDataType(){
			DataType dataType = derivedField.getDataType();
			if(dataType == null){
				throw new MissingAttributeException(derivedField, org.dmg.pmml.PMMLAttributes.DERIVEDFIELD_DATATYPE);
			}

			return dataType;
		}

		@Override
		public OpType getOpType(){
			OpType opType = derivedField.getOpType();
			if(opType == null){
				throw new MissingAttributeException(derivedField, org.dmg.pmml.PMMLAttributes.DERIVEDFIELD_OPTYPE);
			}

			return opType;
		}

		@Override
		public List<?> getOrdering(){
			List<?> ordering = FieldUtil.getValidValues(derivedField);

			return ordering;
		}
	};

	return FieldValueUtil.create(typeInfo, value);
}
 

public DerivedField getDerivedField(){
	return this.derivedField;
}