• Example Search
• Project Search

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• aardpfark-master
  • src
    • main
      • scala
        • sparktypes.scala
        • com
          • ibm
            • aardpfark
              • avro
                • SchemaConverters.scala
              • spark
                • ml
                  • classification
                    • DecisionTreeClassificationModel.scala
                    • MLPClassifier.scala
                    • RandomForestClassificationModel.scala
                    • LogisticRegressionModel.scala
                    • NaiveBayesModel.scala
                    • LinearSVCModel.scala
                    • GBTClassificationModel.scala
                  • PFATransformers.scala
                  • clustering
                    • KMeans.scala
                  • linear
                    • LinearModelData.scala
                    • SparkLinearModel.scala
                  • Merge.scala
                  • tree
                    • Trees.scala
                  • regression
                    • GBTRegressionModel.scala
                    • RandomForestRegressionModel.scala
                    • GeneralizedLinearRegressionModel.scala
                    • LinearRegressionModel.scala
                    • DecisionTreeRegressionModel.scala
                  • feature
                    • Binarizer.scala
                    • StringIndexerModel.scala
                    • StopWordsRemover.scala
                    • MaxAbsScaler.scala
                    • Bucketizer.scala
                    • ElementwiseProduct.scala
                    • MinMaxScaler.scala
                    • VectorSelector.scala
                    • PCAModel.scala
                    • Normalizer.scala
                    • CountVectorizerModel.scala
                    • RegexTokenizer.scala
                    • ChiSqSelectorModel.scala
                    • VectorAssembler.scala
                    • NGram.scala
                    • VectorSlicer.scala
                    • StandardScaler.scala
                    • IDF.scala
                  • SparkSupport.scala
              • pfa
                • functions
                  • package.scala
                  • FunctionLibrary.scala
                • utils
                  • Util.scala
                • package.scala
                • document
                  • JSONSerializers.scala
                  • PFABuilder.scala
                  • Cell.scala
                  • PFADocument.scala
                • types
                  • package.scala
                  • WithSchema.scala
                • expression
                  • LetSet.scala
                  • FunctionRefs.scala
                  • ControlStructures.scala
                  • Casts.scala
                  • package.scala
                  • PFAExpression.scala
                  • FunctionCalls.scala
                  • New.scala
                  • AttributeRetrieval.scala
                  • Loops.scala
    • test
      • scala
        • com
          • ibm
            • aardpfark
              • spark
                • ml
                  • classification
                    • DecisionTreeClassifierSuite.scala
                    • MLPClassifierSuite.scala
                    • SparkClassifierPFASuiteBase.scala
                    • GBTClassificationSuite.scala
                    • RandomForestClassificationSuite.scala
                    • NaiveBayesSuite.scala
                    • LinearSVCSuite.scala
                    • LogisticRegressionSuite.scala
                  • SparkMLTestUtils.scala
                  • clustering
                    • KMeansModelSuite.scala
                  • PredictorPipelineSuite.scala
                  • regression
                    • SparkRegressorPFASuiteBase.scala
                    • LinearRegressionSuite.scala
                    • DecisionTreeRegressionSuite.scala
                    • GeneralizedLinearRegressionSuite.scala
                    • RandomForestRegressionSuite.scala
                    • GBTRegressionSuite.scala
                  • feature
                    • MinMaxScalerSuite.scala
                    • NormalizerSuite.scala
                    • NGramSuite.scala
                    • StopWordsRemoverSuite.scala
                    • VectorAssemblerSuite.scala
                    • ElementwiseProductSuite.scala
                    • VectorSlicerSuite.scala
                    • RegexTokenizerSuite.scala
                    • PCAModelSuite.scala
                    • ChiSqSelectorSuite.scala
                    • IDFSuite.scala
                    • StringIndexerModelSuite.scala
                    • StandardScalerSuite.scala
                    • CountVectorizerSuite.scala
                    • MaxAbsScalerSuite.scala
                    • BucketizerSuite.scala
                    • BinarizerSuite.scala
              • pfa
                • functions
                  • StringLibrarySuite.scala
                  • ImputeLibrarySuite.scala
                  • MapLibrarySuite.scala
                  • LinearAlgebraLibrarySuite.scala
                  • MathLibrarySuite.scala
                  • RegexLibrarySuite.scala
                  • CastLibrarySuite.scala
                  • FunctionLibrarySuite.scala
                  • ArrayLibrarySuite.scala
                  • CoreLibrarySuite.scala
                • SparkPFASuiteBase.scala
                • SparkFeaturePFASuiteBase.scala
                • SparkPredictorPFASuiteBase.scala
                • DSLSuiteBase.scala
                • PFATestUtils.scala
                • expression
                  • FunctionSuite.scala
                  • CastsSuite.scala
                  • LetSetSuite.scala
                  • AttrSuite.scala
                  • ControlStructuresSuite.scala
                  • LoopsSuite.scala
  • MAINTAINERS.md
  • LICENSE
  • project
    • build.properties
    • plugins.sbt
  • CONTRIBUTING.md
  • python
    • README.md
  • data
    • sample_libsvm_data.txt
    • sample_multiclass_classification_data.txt
    • sample_kmeans_data.txt
    • sample_binary_classification_data.txt
    • sample_linear_regression_data.txt
    • sample_lda_libsvm_data.txt
  • .travis.yml
  • README.md
  • build.sbt
  • .gitignore

package com.ibm.aardpfark.spark.ml.feature

import com.ibm.aardpfark.pfa.document.{Cell, PFABuilder, PFADocument}
import com.ibm.aardpfark.pfa.expression._
import com.ibm.aardpfark.pfa.types.WithSchema
import com.ibm.aardpfark.spark.ml.PFAModel
import com.sksamuel.avro4s.{AvroNamespace, AvroSchema}
import org.apache.avro.SchemaBuilder

import org.apache.spark.ml.feature.StandardScalerModel

@AvroNamespace("com.ibm.aardpfark.exec.spark.ml.feature")
case class StandardScalerModelData(mean: Seq[Double], std: Seq[Double]) extends WithSchema {
  def schema = AvroSchema[this.type]
}

class PFAStandardScalerModel(override val sparkTransformer: StandardScalerModel) extends PFAModel[StandardScalerModelData] {
  import com.ibm.aardpfark.pfa.dsl._
  import com.ibm.aardpfark.pfa.dsl.core._

  private val inputCol = sparkTransformer.getInputCol
  private val outputCol = sparkTransformer.getOutputCol
  private val inputExpr = StringExpr(s"input.${inputCol}")

  // references to cell variables
  private val meanRef = modelCell.ref("mean")
  private val stdRef = modelCell.ref("std")

  override def inputSchema = {
    SchemaBuilder.record(withUid(inputBaseName)).fields()
      .name(inputCol).`type`().array().items().doubleType().noDefault()
      .endRecord()
  }

  override def outputSchema = {
    SchemaBuilder.record(withUid(outputBaseName)).fields()
      .name(outputCol).`type`().array().items().doubleType().noDefault()
      .endRecord()
  }

  override def cell = {
    val scalerData = StandardScalerModelData(sparkTransformer.mean.toArray, sparkTransformer.std.toArray)
    Cell(scalerData)
  }

  def partFn(name: String, p: Seq[String], e: PFAExpression) = {
    NamedFunctionDef(name, FunctionDef[Double, Double](p, Seq(e)))
  }

  // function schema
  val (scaleFnDef, scaleFnRef) = if (sparkTransformer.getWithMean) {
    if (sparkTransformer.getWithStd) {
      val meanStdScale = partFn("meanStdScale", Seq("i", "m", "s"), div(minus("i", "m"), "s"))
      (Some(meanStdScale), a.zipmap(inputExpr, meanRef, stdRef, meanStdScale.ref))
    } else {
      val meanScale = partFn("meanScale", Seq("i", "m"), minus("i", "m"))
      (Some(meanScale), a.zipmap(inputExpr, meanRef, meanScale.ref))
    }
  } else {
    if (sparkTransformer.getWithStd) {
      val stdScale = partFn("stdScale", Seq("i", "s"), div("i", "s"))
      (Some(stdScale), a.zipmap(inputExpr, stdRef, stdScale.ref))
    } else {
      (None, inputExpr)
    }
  }

  override def action: PFAExpression = {
    NewRecord(outputSchema, Map(outputCol -> scaleFnRef))
  }

  override def pfa: PFADocument = {
    val builder = PFABuilder()
      .withName(sparkTransformer.uid)
      .withMetadata(getMetadata)
      .withInput(inputSchema)
      .withOutput(outputSchema)
      .withCell(modelCell)
      .withAction(action)
    scaleFnDef.foreach(fnDef => builder.withFunction(fnDef))
    builder.pfa
  }
}