org.scalactic.TolerantNumerics Scala Examples

package com.spotify.noether

import org.scalactic.TolerantNumerics

class AUCTest extends AggregatorTest {
  private implicit val doubleEq = TolerantNumerics.tolerantDoubleEquality(0.1)

  private val data =
    List(
      (0.1, false),
      (0.1, true),
      (0.4, false),
      (0.6, false),
      (0.6, true),
      (0.6, true),
      (0.8, true)
    ).map { case (s, pred) => Prediction(pred, s) }

  it should "return ROC AUC" in {
    assert(run(AUC(ROC, samples = 50))(data) === 0.7)
  }

  it should "return PR AUC" in {
    assert(run(AUC(PR, samples = 50))(data) === 0.83)
  }

  it should "return points of a PR Curve" in {
    val expected = Array(
      (0.0, 1.0),
      (0.0, 1.0),
      (0.25, 1.0),
      (0.75, 0.75),
      (0.75, 0.6),
      (1.0, 0.5714285714285714)
    ).map { case (a, b) => MetricCurvePoint(a, b) }
    assert(run(Curve(PR, samples = 5))(data).points === MetricCurvePoints(expected).points)
  }

  it should "return points of a ROC Curve" in {
    val expected = Array(
      (0.0, 0.0),
      (0.0, 0.25),
      (0.3333333333333333, 0.75),
      (0.6666666666666666, 0.75),
      (1.0, 1.0),
      (1.0, 1.0)
    ).map { case (a, b) => MetricCurvePoint(a, b) }
    assert(run(Curve(ROC, samples = 5))(data).points === MetricCurvePoints(expected).points)
  }
} 

package org.squbs.pattern.timeoutpolicy

import org.scalactic.TolerantNumerics
import org.scalatest.{Matchers, FlatSpecLike}
import org.apache.commons.math3.special.Erf


class MathUtilSpec extends FlatSpecLike with Matchers{

  it should "pass NaN cases" in {
    java.lang.Double.isNaN(MathUtil.erfInv(-1.001)) should be(true)
    java.lang.Double.isNaN(MathUtil.erfInv(1.001)) should be(true)
  }

  it should "pass Infinite case" in {
    MathUtil.erfInv(1) should be(Double.PositiveInfinity)
    MathUtil.erfInv(-1) should be(Double.NegativeInfinity)
  }

  it should "pass regular case" in {
    for (x <- (-5.9).until(5.9, 0.01)) {
      val y = Erf.erf(x)
      val dydx = 2 * math.exp(-x * x) / math.sqrt(Math.PI)
      implicit val equality = TolerantNumerics.tolerantDoubleEquality(1.0e-15 / dydx)
      x shouldEqual(MathUtil.erfInv(y))
    }
  }
} 

package io.picnicml.doddlemodel.linear

import breeze.linalg.{DenseMatrix, DenseVector}
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.{Features, RealVector, Target}
import io.picnicml.doddlemodel.linear.SoftmaxClassifier.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class SoftmaxClassifierTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-3f)

  "Softmax classifier" should "calculate the value of the loss function" in {
    val w = DenseVector(1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 1.0f)
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f),
      List(-2.0f, 1.0f, 0.0f)
    )
    val y = DenseVector(1.0f, 0.0f, 2.0f)

    val model = ev.copy(SoftmaxClassifier(lambda = 1.0f), numClasses = 3)
    ev.lossStateless(model, w, x, y) shouldEqual 19.843778223530194f
  }

  it should "calculate the gradient of the loss function wrt. to model parameters" in {
    for (_ <- 1 to 1000) {
      val w = DenseVector.rand[Float](5 * 9, rand = randomUniform)
      val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
      val y = DenseVector.rangeF(0, 10)
      testGrad(w, x, y)
    }

    def testGrad(w: RealVector, x: Features, y: Target) = {
      val model = ev.copy(SoftmaxClassifier(lambda = 0.5f), numClasses = 10)
      breezeEqual(
        gradApprox(w => ev.lossStateless(model, w, x, y), w),
        ev.lossGradStateless(model, w, x, y)
      ) shouldEqual true
    }
  }

  it should "prevent the usage of negative L2 regularization strength" in {
    an [IllegalArgumentException] shouldBe thrownBy(SoftmaxClassifier(lambda = -0.5f))
  }
} 

package io.picnicml.doddlemodel.linear

import breeze.linalg.{DenseMatrix, DenseVector, convert}
import breeze.stats.distributions.Rand
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.{Features, RealVector, Target}
import io.picnicml.doddlemodel.linear.PoissonRegression.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class PoissonRegressionTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-2f)

  "Poisson regression" should "calculate the value of the loss function" in {
    val w = DenseVector(1.0f, 2.0f, 3.0f)
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f)
    )
    val y = DenseVector(3.0f, 4.0f)

    val model = PoissonRegression(lambda = 1.0f)
    ev.lossStateless(model, w, x, y) shouldEqual 29926.429998513137f
  }

  it should "calculate the gradient of the loss function wrt. to model parameters" in {
    for (_ <- 1 to 1000) {
      val w = DenseVector.rand[Float](5, rand = randomUniform)
      val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
      val y = convert(DenseVector.rand(10, rand = Rand.randInt(20)), Float)
      testGrad(w, x, y)
    }

    def testGrad(w: RealVector, x: Features, y: Target) = {
      val model = PoissonRegression(lambda = 0.5f)
      breezeEqual(
        gradApprox(w => ev.lossStateless(model, w, x, y), w),
        ev.lossGradStateless(model, w, x, y)
      ) shouldEqual true
    }
  }

  it should "prevent the usage of negative L2 regularization strength" in {
    an [IllegalArgumentException] shouldBe thrownBy(PoissonRegression(lambda = -0.5f))
  }

  it should "throw an exception if fitting a model on a dataset that is not count data" in {
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f),
      List(3.0f, 1.0f, 2.0f)
    )
    val y = DenseVector.rand[Float](3, rand = randomUniform)
    val model = PoissonRegression()

    an [IllegalArgumentException] shouldBe thrownBy(ev.fit(model, x, y))
  }
} 

package io.picnicml.doddlemodel.linear

import breeze.linalg.{DenseMatrix, DenseVector}
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.{Features, RealVector, Target}
import io.picnicml.doddlemodel.linear.LinearRegression.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class LinearRegressionTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-3f)

  "Linear regression" should "calculate the value of the loss function" in {
    val w = DenseVector(1.0f, 2.0f, 3.0f)
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f)
    )
    val y = DenseVector(3.0f, 4.0f)

    val model = LinearRegression(lambda = 1)
    ev.lossStateless(model, w, x, y) shouldEqual 24.75f
  }

  it should "calculate the gradient of the loss function wrt. to model parameters" in {
    for (_ <- 1 to 1000) {
      val w = DenseVector.rand[Float](5, rand = randomUniform)
      val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
      val y = DenseVector.rand[Float](10, rand = randomUniform)
      testGrad(w, x, y)
    }

    def testGrad(w: RealVector, x: Features, y: Target) = {
      val model = LinearRegression(lambda = 0.5f)
      breezeEqual(
        gradApprox(w => ev.lossStateless(model, w, x, y), w),
        ev.lossGradStateless(model, w, x, y)
      ) shouldEqual true
    }
  }

  it should "prevent the usage of negative L2 regularization strength" in {
    an [IllegalArgumentException] shouldBe thrownBy(LinearRegression(lambda = -0.5f))
  }
} 

package io.picnicml.doddlemodel.linear

import breeze.linalg.{DenseMatrix, DenseVector, convert}
import breeze.numerics.round
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.{Features, RealVector, Target}
import io.picnicml.doddlemodel.linear.LogisticRegression.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class LogisticRegressionTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-3f)

  "Logistic regression" should "calculate the value of the loss function" in {
    val w = DenseVector(1.0f, 2.0f, 3.0f)
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f)
    )
    val y = DenseVector(1.0f, 0.0f)

    val model = LogisticRegression(lambda = 1)
    ev.lossStateless(model, w, x, y) shouldEqual 7.1566391945397703f
  }

  it should "calculate the gradient of the loss function wrt. to model parameters" in {
    for (_ <- 1 to 1000) {
      val w = DenseVector.rand[Float](5, rand = randomUniform)
      val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
      val y = convert(round(DenseVector.rand[Float](10, rand = randomUniform)), Float)
      testGrad(w, x, y)
    }

    def testGrad(w: RealVector, x: Features, y: Target) = {
      val model = LogisticRegression(lambda = 0.5f)
      breezeEqual(
        gradApprox(w => ev.lossStateless(model, w, x, y), w),
        ev.lossGradStateless(model, w, x, y)
      ) shouldEqual true
    }
  }


  it should "prevent the usage of negative L2 regularization strength" in {
    an [IllegalArgumentException] shouldBe thrownBy(LogisticRegression(lambda = -0.5f))
  }

  it should "throw an exception if fitting a model on a dataset with more than two classes" in {
    val x = DenseMatrix(
      List(3.0f, 1.0f, 2.0f),
      List(-1.0f, -2.0f, 2.0f),
      List(3.0f, 1.0f, 2.0f)
    )
    val y = DenseVector(1.0f, 0.0f, 2.0f)
    val model = LogisticRegression()

    an [IllegalArgumentException] shouldBe thrownBy(ev.fit(model, x, y))
  }
} 

package io.picnicml.doddlemodel.preprocessing

import breeze.linalg.{*, DenseMatrix, DenseVector, convert}
import breeze.stats.{mean, stddev}
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.Feature.{CategoricalFeature, FeatureIndex, NumericalFeature}
import io.picnicml.doddlemodel.preprocessing.StandardScaler.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class StandardScalerTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-4f)

  "Standard scaler" should "preprocess the numerical features" in {
    val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
    val featureIndex = FeatureIndex(
      List(
        NumericalFeature,
        NumericalFeature,
        NumericalFeature,
        NumericalFeature,
        CategoricalFeature
      )
    )
    val scaler = StandardScaler(featureIndex)
    val trainedScaler = ev.fit(scaler, x)
    val xTransformed = ev.transform(trainedScaler, x)

    breezeEqual(mean(x(::, *)).t, DenseVector.zeros[Float](5)) shouldBe false
    breezeEqual(convert(stddev(x(::, *)).t, Float), DenseVector.ones[Float](5)) shouldBe false

    val expectedMeans = DenseVector.zeros[Float](5)
    expectedMeans(-1) = mean(x(::, -1))
    breezeEqual(mean(xTransformed(::, *)).t, expectedMeans) shouldBe true

    val expectedStdDevs = DenseVector.ones[Float](5)
    expectedStdDevs(-1) = stddev(x(::, -1)).toFloat
    breezeEqual(convert(stddev(xTransformed(::, *)).t, Float), expectedStdDevs) shouldBe true
  }

  it should "handle the zero variance case" in {
    val x = DenseMatrix.ones[Float](10, 5)
    val scaler = StandardScaler(FeatureIndex.numerical(5))
    val trainedScaler = ev.fit(scaler, x)
    val xTransformed = ev.transform(trainedScaler, x)

    xTransformed.forall(_.isNaN) shouldBe false
  }

  it should "preprocess a subset of numerical features" in {
    val x = DenseMatrix.rand[Float](10, 5, rand = randomUniform)
    val scaler = StandardScaler(FeatureIndex.numerical(5).subset("f0", "f2", "f4"))
    val trainedScaler = ev.fit(scaler, x)
    val xTransformed = ev.transform(trainedScaler, x)

    breezeEqual(mean(x(::, *)).t, DenseVector.zeros[Float](5)) shouldBe false
    breezeEqual(convert(stddev(x(::, *)).t, Float), DenseVector.ones[Float](5)) shouldBe false

    assert(tolerance.areEqual(mean(xTransformed(::, 0)), 0.0f))
    assert(tolerance.areEqual(convert(stddev(xTransformed(::, 0)), Float), 1.0f))
    assert(!tolerance.areEqual(mean(xTransformed(::, 1)), 0.0f))
    assert(!tolerance.areEqual(convert(stddev(xTransformed(::, 1)), Float), 1.0f))
    assert(tolerance.areEqual(mean(xTransformed(::, 2)), 0.0f))
    assert(tolerance.areEqual(convert(stddev(xTransformed(::, 2)), Float), 1.0f))
    assert(!tolerance.areEqual(mean(xTransformed(::, 3)), 0.0f))
    assert(!tolerance.areEqual(convert(stddev(xTransformed(::, 3)), Float), 1.0f))
    assert(tolerance.areEqual(mean(xTransformed(::, 4)), 0.0f))
    assert(tolerance.areEqual(convert(stddev(xTransformed(::, 4)), Float), 1.0f))
  }
} 

package io.picnicml.doddlemodel.preprocessing

import breeze.linalg.{DenseMatrix, DenseVector}
import io.picnicml.doddlemodel.TestingUtils
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class NormsTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-4f)

  private val x = DenseMatrix(
    List(0.0f, 0.0f, 0.0f),
    List(1.0f, 2.0f, 2.0f),
    List(-2.0f, 0.0f, 0.0f)
  )

  "Norms" should "calculate the L2 norm of each row" in {
    val xExpected = DenseVector(0.0f, 3.0f, 2.0f)
    breezeEqual(Norms.L2Norm(x), xExpected) shouldBe true
  }

  "Norms" should "calculate the L1 norm of each row" in {
    val xExpected = DenseVector(0.0f, 5.0f, 2.0f)
    breezeEqual(Norms.L1Norm(x), xExpected) shouldBe true
  }

  "Norms" should "calculate the max norm of each row" in {
    val xExpected = DenseVector(0.0f, 2.0f, 2.0f)
    breezeEqual(Norms.MaxNorm(x), xExpected) shouldBe true
  }

} 

package io.picnicml.doddlemodel.preprocessing

import breeze.linalg.DenseMatrix
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.Feature.{CategoricalFeature, FeatureIndex, NumericalFeature}
import io.picnicml.doddlemodel.preprocessing.RangeScaler.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class RangeScalerTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-4f)

  private val x = DenseMatrix(
    List(-3.0f, 2.0f, 1.0f),
    List(-3.0f, 3.0f, 0.0f),
    List(-3.0f, 0.0f, 0.0f),
    List(-3.0f, 5.0f, 1.0f)
  )

  "Range scaler" should "scale numerical features to specified range" in {
    val featureIndex = FeatureIndex(List(NumericalFeature, NumericalFeature, CategoricalFeature))
    val rangeScaler1 = RangeScaler((0.0f, 1.0f), featureIndex)
    val trainedRangeScaler1 = ev.fit(rangeScaler1, x)
    val rangeScaler2 = RangeScaler((-5.0f, 15.0f), featureIndex)
    val trainedRangeScaler2 = ev.fit(rangeScaler2, x)

    breezeEqual(
      ev.transform(trainedRangeScaler1, x),
      DenseMatrix(
        List(0.0f, 0.4f, 1.0f),
        List(0.0f, 0.6f, 0.0f),
        List(0.0f, 0.0f, 0.0f),
        List(0.0f, 1.0f, 1.0f)
      )
    ) shouldBe true

    breezeEqual(
      ev.transform(trainedRangeScaler2, x),
      DenseMatrix(
        List(-5.0f, 3.0f, 1.0f),
        List(-5.0f, 7.0f, 0.0f),
        List(-5.0f, -5.0f, 0.0f),
        List(-5.0f, 15.0f, 1.0f)
      )
    ) shouldBe true
  }

  it should "scale selected subset of numerical features to specified range" in {
    val featureIndex = FeatureIndex(List(NumericalFeature, NumericalFeature, CategoricalFeature))
    val rangeScaler1 = RangeScaler((0.0f, 1.0f), featureIndex.subset(1 to 1))
    val trainedRangeScaler1 = ev.fit(rangeScaler1, x)
    val rangeScaler2 = RangeScaler((-5.0f, 15.0f), featureIndex.subset(1 to 1))
    val trainedRangeScaler2 = ev.fit(rangeScaler2, x)

    breezeEqual(
      ev.transform(trainedRangeScaler1, x),
      DenseMatrix(
        List(-3.0f, 0.4f, 1.0f),
        List(-3.0f, 0.6f, 0.0f),
        List(-3.0f, 0.0f, 0.0f),
        List(-3.0f, 1.0f, 1.0f)
      )
    ) shouldBe true

    breezeEqual(
      ev.transform(trainedRangeScaler2, x),
      DenseMatrix(
        List(-3.0f, 3.0f, 1.0f),
        List(-3.0f, 7.0f, 0.0f),
        List(-3.0f, -5.0f, 0.0f),
        List(-3.0f, 15.0f, 1.0f)
      )
    ) shouldBe true
  }

  it should "amount to no-op if there are no numerical features in data" in {
    val featureIndex = FeatureIndex(List(CategoricalFeature, CategoricalFeature, CategoricalFeature))
    val rangeScaler = RangeScaler((0.0f, 1.0f), featureIndex)
    val trainedRangeScaler = ev.fit(rangeScaler, x)

    breezeEqual(ev.transform(trainedRangeScaler, x), x) shouldBe true
  }
} 

package io.picnicml.doddlemodel.preprocessing

import breeze.linalg.DenseMatrix
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.preprocessing.Normalizer.ev
import io.picnicml.doddlemodel.preprocessing.Norms.{L1Norm, MaxNorm}
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class NormalizerTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-4f)

  "Normalizer" should "scale rows to unit norm using various norms" in {
    val x = DenseMatrix(
      List(1.0f, 2.0f, 2.0f),
      List(-1.0f, 1.0f, 0.5f),
      List(-2.0f, 0.0f, 0.0f)
    )
    val l2Normalizer = Normalizer()
    val l1Normalizer = Normalizer(L1Norm)
    val maxNormalizer = Normalizer(MaxNorm)

    breezeEqual(ev.transform(l2Normalizer, x),
      DenseMatrix(
        List(0.3333f, 0.6666f, 0.6666f),
        List(-0.6666f, 0.6666f, 0.3333f),
        List(-1.0f, 0.0f, 0.0f)
      )
    ) shouldBe true

    breezeEqual(ev.transform(l1Normalizer, x),
      DenseMatrix(
        List(0.2f, 0.4f, 0.4f),
        List(-0.4f, 0.4f, 0.2f),
        List(-1.0f, 0.0f, 0.0f)
      )
    ) shouldBe true

    breezeEqual(ev.transform(maxNormalizer, x),
      DenseMatrix(
        List(0.5f, 1.0f, 1.0f),
        List(-1.0f, 1.0f, 0.5f),
        List(-1.0f, 0.0f, 0.0f)
      )
    ) shouldBe true
  }

  it should "handle rows with zero norm" in {
    val l2Normalizer = Normalizer()
    val x = DenseMatrix(
      List(0.0f, 0.0f, 0.0f),
      List(0.0f, 3.0f, 4.0f)
    )
    val xNormalizedExpected = DenseMatrix(
      List(0.0f, 0.0f, 0.0f),
      List(0.0f, 0.6f, 0.8f)
    )

    breezeEqual(ev.transform(l2Normalizer, x), xNormalizedExpected) shouldBe true
  }
} 

package io.picnicml.doddlemodel.dummy.classification

import breeze.linalg.{DenseVector, convert}
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.{loadBreastCancerDataset, loadIrisDataset}
import io.picnicml.doddlemodel.dummy.classification.StratifiedClassifier.ev
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class StratifiedClassifierTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1e-3f)

  "Stratified classifier" should "infer a categorical distribution from the iris dataset" in {
    val (x, y, _) = loadIrisDataset
    val model = StratifiedClassifier()
    val trainedModel = ev.fit(model, x, y)
    breezeEqual(
      convert(trainedModel.getTargetDistributionParams, Float),
      DenseVector(0.333f, 0.333f, 0.333f)
    ) shouldBe true
  }

  it should "infer a categorical distribution from the breast cancer dataset" in {
    val (x, y, _) = loadBreastCancerDataset
    val model = StratifiedClassifier()
    val trainedModel = ev.fit(model, x, y)
    breezeEqual(
      convert(trainedModel.getTargetDistributionParams, Float),
      DenseVector(0.372f, 0.627f)
    ) shouldBe true
  }
} 

package io.picnicml.doddlemodel.data

import breeze.linalg.DenseVector
import io.picnicml.doddlemodel.TestingUtils
import io.picnicml.doddlemodel.data.DatasetUtils.{shuffleDataset, splitDataset, splitDatasetWithGroups}
import org.scalactic.{Equality, TolerantNumerics}

import scala.util.Random
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class DatasetUtilsTest extends AnyFlatSpec with Matchers with TestingUtils {

  implicit val rand: Random = new Random(0)
  implicit val tolerance: Equality[Float] = TolerantNumerics.tolerantFloatEquality(1.0f)

  val (x, y, _) = loadIrisDataset

  "Dataset utils" should "shuffle the dataset" in {
    val (_, yShuffled) = shuffleDataset(x, y)
    breezeEqual(y, yShuffled) shouldBe false
  }

  they should "split the dataset" in {
    val split = splitDataset(x, y)
    split.yTr.length shouldBe 75
    split.yTe.length shouldBe 75
  }

  they should "split the dataset with groups" in {
    val groups = DenseVector((0 until x.rows).map(x => x % 4):_*)
    val split = splitDatasetWithGroups(x, y, groups, proportionTrain = 0.8f)
    val groupsTe = split.groupsTe.toArray
    split.groupsTr.forall(trGroup => !groupsTe.contains(trGroup)) shouldBe true
  }
} 

package com.github.jongwook

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.{FlatSpec, Matchers}


class TestPerfectScores extends FlatSpec with Matchers {
  import TestFixture._
  implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(eps)

  val perfectScores: Seq[(String, Map[Metric, Seq[Double]])] = {
    val spark = SparkSession.builder().master(new SparkConf().get("spark.master", "local[8]")).getOrCreate()

    import spark.implicits._

    val predictionDF = spark.createDataset(prediction)
    val groundTruthDF = spark.createDataset(groundTruth)

    for ((name, df) <- Seq("prediction" -> predictionDF, "ground truth" -> groundTruthDF)) yield {
      val metrics = new SparkRankingMetrics(df, df, itemCol = "product", predictionCol = "rating")

      name -> Map[Metric, Seq[Double]](
        NDCG -> metrics.ndcgAt(ats),
        MAP -> metrics.mapAt(ats),
        Precision -> metrics.precisionAt(Seq(Integer.MAX_VALUE)),
        Recall -> metrics.recallAt(Seq(Integer.MAX_VALUE))
      )
    }
  }

  for ((name, scores) <- perfectScores) {
    for (metric <- Seq(NDCG, MAP, Precision, Recall)) {
      s"In $name dataset, our $metric implementation" should s"return 1.0 for the perfect prediction" in {
        for (score <- scores(metric)) {
          score should equal(1.0)
        }
      }
    }
  }
} 

package ml.combust.mleap.core.feature

import ml.combust.mleap.core.types.{BasicType, ListType, StructField, TensorType}
import org.apache.spark.ml.linalg.Vectors
import org.scalactic.TolerantNumerics
import org.scalatest.FunSpec

class WordToVectorModelSpec extends FunSpec {
  implicit val doubleEquality = TolerantNumerics.tolerantDoubleEquality(0.000001)

  describe("word to vector model") {
    val model = WordToVectorModel(Map("test" -> 1), Array(12))

    it("has the right input schema") {
      assert(model.inputSchema.fields ==
        Seq(StructField("input", ListType(BasicType.String))))
    }

    it("has the right output schema") {
      assert(model.outputSchema.fields ==
        Seq(StructField("output", TensorType.Double(1))))
    }
  }

  describe("WordToVectorKernel") {
    describe("for name") {
      it("returns the kernel for string") {
        assert(WordToVectorKernel.forName("default") == WordToVectorKernel.Default)
        assert(WordToVectorKernel.forName("sqrt") == WordToVectorKernel.Sqrt)
      }
    }
  }

  describe("Sqrt kernel") {
    it("produces results using the sqrt kernel (division by sqrt(dot(vec, vec)))") {
      val hello = Vectors.dense(-0.02743354,  0.13925314, -0.41874424,  0.05635237, -1.01364303,
        0.13555442, -0.36437142,  0.10494551,  1.25634718,  0.74919909,
        -0.75405639,  0.34798685, -0.33082211, -1.83296537,  1.8524611 ,
        0.16053002,  0.05308712, -0.61047131, -2.04251647, -0.6457383 ,
        -0.06899478, -1.06984603,  1.81890905, -1.57762015, -1.14214861,
        -0.37704349, -1.13758969, -1.11241293, -0.01736556,  0.55350637,
        1.29117298,  0.6780861 ,  0.72507775,  0.38882053, -1.13152575)
      val there = Vectors.dense(0.05639598, -0.0189869 ,  0.01236993,  0.00477022, -0.10707449,
        0.02502576,  0.0702049 ,  0.07715208,  0.03785434,  0.06749821,
        0.0028507 ,  0.03143736, -0.07800865, -0.066576  ,  0.05038944,
        0.04129622,  0.05770208, -0.09861612, -0.02329824, -0.03803944,
        -0.01226865, -0.03243028,  0.05924392, -0.07248155, -0.03818463,
        0.03131858, -0.03253553,  0.04506788, -0.02503723, -0.03580079,
        0.05802456, -0.00171577, -0.07222789,  0.01021192,  0.01579604)
      val `{make}` = Vectors.dense(1.69664776, -0.9033435 , -1.13164949,  1.94182444, -0.53111398,
        2.28728724,  1.39580894,  1.38314795, -1.03503716,  1.0247947 ,
        -2.175174  ,  1.62514234, -0.64084077, -0.20218629, -0.0694286 ,
        0.37854579, -2.70390058, -2.27423668, -2.79813218, -0.46218753,
        0.77630186, -0.82613772,  1.18320072, -2.93088889,  0.6440177 ,
        -0.02956525, -1.51469374, -2.94850779, -0.89843947, -0.16953184,
        -1.4054004 , -1.22051024,  0.41841957,  0.26196802,  3.39272285)
      val wordVectors = Array(hello, there, `{make}`).flatMap(_.toArray)

      val model = WordToVectorModel(Map("hello" -> 0, "there" -> 1, "{make}" -> 2),
        wordVectors,
        kernel = WordToVectorKernel.Sqrt)

      val resultHello = model(Seq("hello"))
      val expectedHello = Vectors.dense(-0.00489383,  0.02484115, -0.07469912,  0.01005261, -0.18082216,
        0.02418134, -0.06499964,  0.01872106,  0.22411777,  0.13364843,
        -0.13451492,  0.06207682, -0.05901483, -0.32697977,  0.33045758,
        0.02863669,  0.00947013, -0.108901  , -0.36436126, -0.11519223,
        -0.01230787, -0.19084813,  0.32447228, -0.28142914, -0.20374607,
        -0.06726019, -0.20293281, -0.19844157, -0.00309781,  0.09873912,
        0.23033029,  0.1209627 ,  0.12934546,  0.06936107, -0.20185107)

      val resultSentence = model(Seq("hello", "there", "{make}", "qqq"))
      val expectedSentence = Vectors.dense(0.13878191, -0.06297886, -0.1236953 ,  0.16108668, -0.13284827,
        0.19686932,  0.0885994 ,  0.12588461,  0.02084325,  0.14810168,
        -0.23535359,  0.16121693, -0.08441966, -0.16903109,  0.14745265,
        0.04667632, -0.20855054, -0.23993334, -0.39118211, -0.09216406,
        0.05589835, -0.15509237,  0.24620885, -0.36842539, -0.04313309,
        -0.03018265, -0.21592611, -0.32297428, -0.07566708,  0.02800181,
        -0.00452011, -0.04376236,  0.08615666,  0.05316085,  0.18312679)
      for ((a, b) <- resultHello.toArray.zip(expectedHello.toArray)) { assert(a === b) }
      for ((a, b) <- resultSentence.toArray.zip(expectedSentence.toArray)) { assert(a === b) }
    }
  }
} 

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{Matchers, WordSpec}

class EqualitySpec extends WordSpec with Matchers with TypeCheckedTripleEquals{
  implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001)

  implicit def vectorEquality[A](implicit eqA: Equality[A]): Equality[Vector[A]] = new Equality[Vector[A]] {
    override def areEqual(v1: Vector[A], b: Any): Boolean = b match {
      case v2: Vector[_] =>
        v1.size == v2.size &&
          v1.zip(v2).forall { case ((x, y)) => eqA.areEqual(x, y)}
      case _ => false
    }
  }

  "Equality" should {
    "allow to compare two Double with a tolerance" in {
      1.6 + 1.8 should ===(3.4)
    }

    "allow to compare two Vector[Double] with a tolerance" in {
      Vector(1.6 + 1.8, 0.0051) should === (Vector(3.4, 0.0052))
    }
  }
} 

package com.spotify.noether

import org.scalactic.{Equality, TolerantNumerics}

class LogLossTest extends AggregatorTest {
  private implicit val doubleEq: Equality[Double] =
    TolerantNumerics.tolerantDoubleEquality(0.1)
  private val classes = 10
  private def s(idx: Int, score: Double): List[Double] =
    0.until(classes).map(i => if (i == idx) score else 0.0).toList

  it should "return correct scores" in {
    val data = List((s(0, 0.8), 0), (s(1, 0.6), 1), (s(2, 0.7), 2)).map {
      case (scores, label) => Prediction(label, scores)
    }

    assert(run(LogLoss)(data) === 0.363548039673)
  }
} 

package com.spotify.noether

import breeze.linalg.DenseMatrix
import org.scalactic.TolerantNumerics

class ConfusionMatrixTest extends AggregatorTest {
  it should "return correct confusion matrix" in {
    val data =
      List(
        (0, 0),
        (0, 0),
        (0, 0),
        (0, 1),
        (0, 1),
        (1, 0),
        (1, 0),
        (1, 0),
        (1, 0),
        (1, 1),
        (1, 1),
        (2, 1),
        (2, 2),
        (2, 2),
        (2, 2)
      ).map { case (p, a) => Prediction(a, p) }

    val labels = Seq(0, 1, 2)
    val actual = run(ConfusionMatrix(labels))(data)

    val mat = DenseMatrix.zeros[Long](labels.size, labels.size)
    mat(0, 0) = 3L
    mat(0, 1) = 2L
    mat(0, 2) = 0L
    mat(1, 0) = 4L
    mat(1, 1) = 2L
    mat(1, 2) = 0L
    mat(2, 0) = 0L
    mat(2, 1) = 1L
    mat(2, 2) = 3L
    assert(actual == mat)
  }

  it should "return correct scores" in {
    val data = List(
      (0, 0),
      (0, 1),
      (0, 0),
      (1, 0),
      (1, 1),
      (1, 1),
      (1, 1)
    ).map { case (s, pred) => Prediction(pred, s) }

    val matrix = run(ConfusionMatrix(Seq(0, 1)))(data)

    assert(matrix(1, 1) === 3L)
    assert(matrix(0, 1) === 1L)
    assert(matrix(1, 0) === 1L)
    assert(matrix(0, 0) === 2L)
  }
} 

package com.spotify.noether

import org.scalactic.TolerantNumerics

class ClassificationReportTest extends AggregatorTest {
  private implicit val doubleEq = TolerantNumerics.tolerantDoubleEquality(0.1)

  it should "return correct scores" in {
    val data = List(
      (0.1, false),
      (0.1, true),
      (0.4, false),
      (0.6, false),
      (0.6, true),
      (0.6, true),
      (0.8, true)
    ).map { case (s, pred) => Prediction(pred, s) }

    val score = run(ClassificationReport())(data)

    assert(score.recall === 0.75)
    assert(score.precision === 0.75)
    assert(score.fscore === 0.75)
    assert(score.fpr === 0.333)
  }

  it should "support multiclass reports" in {
    val predictions = Seq(
      (0, 0),
      (0, 0),
      (0, 1),
      (1, 1),
      (1, 1),
      (1, 0),
      (1, 2),
      (2, 2),
      (2, 2),
      (2, 2)
    ).map { case (p, a) => Prediction(a, p) }

    val reports = run(MultiClassificationReport(Seq(0, 1, 2)))(predictions)

    val report0 = reports(0)
    assert(report0.recall == 2.0 / 3.0)
    assert(report0.precision == 2.0 / 3.0)
    val report1 = reports(1)
    assert(report1.recall == 2.0 / 3.0)
    assert(report1.precision == 0.5)
    val report2 = reports(2)
    assert(report2.recall == 0.75)
    assert(report2.precision == 1.0)
  }
} 

package com.spotify.noether

import org.scalactic.{Equality, TolerantNumerics}

class PrecisionAtKTest extends AggregatorTest {
  import RankingData._
  private implicit val doubleEq: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.1)

  it should "compute precisionAtK for rankings" in {
    assert(run(PrecisionAtK[Int](1))(rankingData) === 1.0 / 3)
    assert(run(PrecisionAtK[Int](2))(rankingData) === 1.0 / 3)
    assert(run(PrecisionAtK[Int](3))(rankingData) === 1.0 / 3)
    assert(run(PrecisionAtK[Int](4))(rankingData) === 0.75 / 3)
    assert(run(PrecisionAtK[Int](5))(rankingData) === 0.8 / 3)
    assert(run(PrecisionAtK[Int](10))(rankingData) === 0.8 / 3)
    assert(run(PrecisionAtK[Int](15))(rankingData) === 8.0 / 45)
  }

  it should "compute precisionAtK for rankings with few predictions" in {
    assert(run(PrecisionAtK[Int](1))(smallRankingData) === 0.5)
    assert(run(PrecisionAtK[Int](2))(smallRankingData) === 0.25)
  }
} 

package com.spotify.noether

import org.scalactic.TolerantNumerics

class ErrorRateSummaryTest extends AggregatorTest {
  private implicit val doubleEq = TolerantNumerics.tolerantDoubleEquality(0.1)
  private val classes = 10
  private def s(idx: Int): List[Double] =
    0.until(classes).map(i => if (i == idx) 1.0 else 0.0).toList

  it should "return correct scores" in {
    val data =
      List((s(1), 1), (s(3), 1), (s(5), 5), (s(2), 3), (s(0), 0), (s(8), 1)).map {
        case (scores, label) => Prediction(label, scores)
      }

    assert(run(ErrorRateSummary)(data) === 0.5)
  }
} 

package com.spotify.noether

import org.scalactic.{Equality, TolerantNumerics}

class CalibrationHistogramTest extends AggregatorTest {
  it should "return correct histogram" in {
    implicit val doubleEq: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.001)
    val data = Seq(
      (0.15, 1.15), // lb
      (0.288, 1.288), // rounding error puts this in (0.249, 0.288)
      (0.30, 1.30), // (0.288, 0.3269)
      (0.36, 1.36), // (0.3269, 0.365)
      (0.555, 1.555), // (0.5219, 0.5609)
      (1.2, 2.2), // ub
      (0.7, 1.7) // ub
    ).map { case (p, a) => Prediction(a, p) }

    val actual = run(CalibrationHistogram(0.21, 0.60, 10))(data)

    val expected = List(
      CalibrationHistogramBucket(Double.NegativeInfinity, 0.21, 1.0, 1.15, 0.15),
      CalibrationHistogramBucket(0.21, 0.249, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.249, 0.288, 1.0, 1.288, 0.288),
      CalibrationHistogramBucket(0.288, 0.327, 1.0, 1.30, 0.30),
      CalibrationHistogramBucket(0.327, 0.366, 1.0, 1.36, 0.36),
      CalibrationHistogramBucket(0.366, 0.405, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.405, 0.4449, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.444, 0.483, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.483, 0.522, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.522, 0.561, 1.0, 1.555, 0.555),
      CalibrationHistogramBucket(0.561, 0.6, 0.0, 0.0, 0.0),
      CalibrationHistogramBucket(0.6, Double.PositiveInfinity, 2.0, 3.9, 1.9)
    )

    assert(actual.length == expected.length)
    (0 until expected.length).foreach { i =>
      assert(actual(i).numPredictions === expected(i).numPredictions)
      assert(actual(i).sumPredictions === expected(i).sumPredictions)
      assert(actual(i).sumLabels === expected(i).sumLabels)
      assert(actual(i).lowerThresholdInclusive === expected(i).lowerThresholdInclusive)
      assert(actual(i).upperThresholdExclusive === expected(i).upperThresholdExclusive)
    }
  }
} 

package com.spotify.noether

import org.scalactic.{Equality, TolerantNumerics}

class NdcgAtKTest extends AggregatorTest {
  import RankingData._
  private implicit val doubleEq: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.1)

  it should "compute ndcg for rankings" in {
    assert(run(NdcgAtK[Int](3))(rankingData) === 1.0 / 3)
    assert(run(NdcgAtK[Int](5))(rankingData) === 0.328788)
    assert(run(NdcgAtK[Int](10))(rankingData) === 0.487913)
    assert(run(NdcgAtK[Int](15))(rankingData) === run(NdcgAtK[Int](10))(rankingData))
  }

  it should "compute ndcg for rankings with few predictions" in {
    assert(run(NdcgAtK[Int](1))(smallRankingData) === 0.5)
    assert(run(NdcgAtK[Int](2))(smallRankingData) === 0.30657)
  }
} 

package com.twosigma.flint.rdd.function.summarize

import com.twosigma.flint.FlintSuite
import com.twosigma.flint.rdd.function.summarize.summarizer.subtractable.{ SumSummarizer => SumSum }
import com.twosigma.flint.rdd.function.summarize.summarizer.subtractable.LeftSubtractableSummarizer
import com.twosigma.flint.rdd.{ KeyPartitioningType, OrderedRDD }
import org.scalactic.TolerantNumerics

case class KVSumSummarizer()
  extends LeftSubtractableSummarizer[(Int, Double), Double, Double] {
  val sum = SumSum[Double]()

  def toT(input: (Int, Double)): Double = input._2

  override def zero(): Double = sum.zero()
  override def add(u: Double, t: (Int, Double)): Double = sum.add(u, toT(t))
  override def subtract(u: Double, t: (Int, Double)): Double =
    sum.subtract(u, toT(t))
  override def merge(u1: Double, u2: Double): Double = sum.merge(u1, u2)
  override def render(u: Double): Double = sum.render(u)
}

class SummarizeSpec extends FlintSuite {

  val data = Array(
    (1000L, (1, 0.01)),
    (1000L, (2, 0.01)),
    (1005L, (1, 0.01)),
    (1005L, (2, 0.01)),
    (1010L, (1, 0.01)),
    (1010L, (2, 0.01)),
    (1015L, (1, 0.01)),
    (1015L, (2, 0.01)),
    (1020L, (1, 0.01)),
    (1020L, (2, 0.01)),
    (1025L, (1, 0.01)),
    (1025L, (2, 0.01)),
    (1030L, (1, 0.01)),
    (1030L, (2, 0.01)),
    (1035L, (1, 0.01)),
    (1035L, (2, 0.01)),
    (1040L, (1, 0.01)),
    (1040L, (2, 0.01)),
    (1045L, (1, 0.01)),
    (1045L, (2, 0.01))
  )

  val summarizer = new KVSumSummarizer()

  var orderedRDD: OrderedRDD[Long, (Int, Double)] = _

  implicit val doubleEq = TolerantNumerics.tolerantDoubleEquality(1.0e-6)

  override def beforeAll() {
    super.beforeAll()
    orderedRDD =
      OrderedRDD.fromRDD(sc.parallelize(data, 4), KeyPartitioningType.Sorted)
  }

  "Summarize" should "apply correctly" in {
    val skFn = { case ((_, _)) => None }: ((Int, Double)) => Option[Nothing]
    (1 to 4).foreach { depth =>
      val ret = Summarize(orderedRDD, summarizer, skFn, depth)
      assert(ret.size == 1)
      assert(ret.head._2 === data.length * 0.01)
    }
  }

  it should "apply with sk correctly" in {
    val skFn = { case ((sk, _)) => sk }: ((Int, Double)) => Int
    (1 to 4).foreach { depth =>
      val ret = Summarize(orderedRDD, summarizer, skFn, depth)
      assert(ret.size == 2)
      assert(ret.head._2 === 0.1)
    }
  }
} 

package retcalc

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{Matchers, WordSpec}

class RetCalcSpec extends WordSpec with Matchers with TypeCheckedTripleEquals {

  implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001)

  "RetCalc.futureCapital" should {
    "calculate the amount of savings I will have in n months" in {
      // Excel =-FV(0.04/12,25*12,1000,10000,0)
      val actual = RetCalc.futureCapital(FixedReturns(0.04), nbOfMonths = 25 * 12,
        netIncome = 3000, currentExpenses = 2000, initialCapital = 10000)
      val expected = 541267.1990
      actual should ===(expected)
    }

    "calculate how much savings will be left after having taken a pension for n months" in {
      val actual = RetCalc.futureCapital(FixedReturns(0.04), nbOfMonths = 40 * 12,
        netIncome = 0, currentExpenses = 2000, initialCapital = 541267.198962)
      val expected = 309867.5316
      actual should ===(expected)
    }
  }

  val params = RetCalcParams(
    nbOfMonthsInRetirement = 40 * 12,
    netIncome = 3000,
    currentExpenses = 2000,
    initialCapital = 10000)

  "RetCalc.simulatePlan" should {
    "calculate the capital at retirement and the capital after death" in {
      val (capitalAtRetirement, capitalAfterDeath) = RetCalc.simulatePlan(
        returns = FixedReturns(0.04), params, nbOfMonthsSavings = 25 * 12)

      capitalAtRetirement should ===(541267.1990)
      capitalAfterDeath should ===(309867.5316)
    }

    "use different returns for capitalisation and drawdown" in {
      val nbOfMonthsSavings = 25 * 12
      val returns = VariableReturns(
        Vector.tabulate(nbOfMonthsSavings + params.nbOfMonthsInRetirement)(i =>
          if (i < nbOfMonthsSavings)
            VariableReturn(i.toString, 0.04 / 12)
          else
            VariableReturn(i.toString, 0.03 / 12)))
      val (capitalAtRetirement, capitalAfterDeath) =
        RetCalc.simulatePlan(returns, params, nbOfMonthsSavings)
      // Excel: =-FV(0.04/12, 25*12, 1000, 10000)
      capitalAtRetirement should ===(541267.1990)
      // Excel: =-FV(0.03/12, 40*12, -2000, 541267.20)
      capitalAfterDeath should ===(-57737.7227)
    }

  }


  "RetCalc.nbOfMonthsSaving" should {
    "calculate how long I need to save before I can retire" in {
      val actual = RetCalc.nbOfMonthsSaving(params, FixedReturns(0.04))
      val expected = 23 * 12 + 1
      actual should ===(expected)
    }

    "not crash if the resulting nbOfMonths is very high" in {
      val actual = RetCalc.nbOfMonthsSaving(
        params = RetCalcParams(
          nbOfMonthsInRetirement = 40 * 12,
          netIncome = 3000, currentExpenses = 2999, initialCapital = 0),
        returns = FixedReturns(0.01))
      val expected = 8280
      actual should ===(expected)
    }

    "not loop forever if I enter bad parameters" in {
      val actual = RetCalc.nbOfMonthsSaving(params.copy(netIncome = 1000), FixedReturns(0.04))
      actual should ===(Int.MaxValue)
    }
  }
} 

package retcalc

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{Matchers, WordSpec}

class ReturnsSpec extends WordSpec with Matchers with TypeCheckedTripleEquals {

  implicit val doubleEquality: Equality[Double] =
    TolerantNumerics.tolerantDoubleEquality(0.0001)

  "Returns.monthlyReturn" should {
    "return a fixed rate for a FixedReturn" in {
      Returns.monthlyRate(FixedReturns(0.04), 0) should ===(0.04 / 12)
      Returns.monthlyRate(FixedReturns(0.04), 10) should ===(0.04 / 12)
    }

    val variableReturns = VariableReturns(
      Vector(VariableReturn("2000.01", 0.1), VariableReturn("2000.02", 0.2)))
    "return the nth rate for VariableReturn" in {
      Returns.monthlyRate(variableReturns, 0) should ===(0.1)
      Returns.monthlyRate(variableReturns, 1) should ===(0.2)
    }

    "roll over from the first rate if n > length" in {
      Returns.monthlyRate(variableReturns, 2) should ===(0.1)
      Returns.monthlyRate(variableReturns, 3) should ===(0.2)
      Returns.monthlyRate(variableReturns, 4) should ===(0.1)
    }

    "return the n+offset th rate for OffsetReturn" in {
      val returns = OffsetReturns(variableReturns, 1)
      Returns.monthlyRate(returns, 0) should ===(0.2)
      Returns.monthlyRate(returns, 1) should ===(0.1)
    }
  }


  "Returns.fromEquityAndInflationData" should {
    "compute real total returns from equity and inflation data" in {
      val equities = Vector(
        EquityData("2117.01", 100.0, 10.0),
        EquityData("2117.02", 101.0, 12.0),
        EquityData("2117.03", 102.0, 12.0))

      val inflations = Vector(
        InflationData("2117.01", 100.0),
        InflationData("2117.02", 102.0),
        InflationData("2117.03", 102.0))

      val returns = Returns.fromEquityAndInflationData(equities, inflations)
      returns should ===(VariableReturns(Vector(
        VariableReturn("2117.02", (101.0 + 12.0 / 12) / 100.0 - 102.0 / 100.0),
        VariableReturn("2117.03", (102.0 + 12.0 / 12) / 101.0 - 102.0 / 102.0))))
    }
  }

  "VariableReturns.fromUntil" should {
    "keep only a window of the returns" in {
      val variableReturns = VariableReturns(Vector.tabulate(12) { i =>
        val d = (i + 1).toDouble
        VariableReturn(f"2017.$d%02.0f", d)
      })

      variableReturns.fromUntil("2017.07", "2017.09").returns should ===(Vector(
        VariableReturn("2017.07", 7.0),
        VariableReturn("2017.08", 8.0)
      ))

      variableReturns.fromUntil("2017.10", "2018.01").returns should ===(Vector(
        VariableReturn("2017.10", 10.0),
        VariableReturn("2017.11", 11.0),
        VariableReturn("2017.12", 12.0)
      ))
    }
  }

  "Returns.annualizedTotalReturn" should {
    val returns = VariableReturns(Vector.tabulate(12)(i => VariableReturn(i.toString, i.toDouble / 100 / 12)))
    val avg = Returns.annualizedTotalReturn(returns)
    "compute a geometric mean of the returns" in {
      // Excel: GEOMEAN (see geomean.ods)
      avg should ===(0.0549505735)
    }

    "compute an average that can be used to calculate a futureCapital instead of using variable returns" in {
      // This calculation only works if the capital does not change over time
      // otherwise, the capital fluctuates as well as the interest rates, and we cannot use the mean
      val futCapVar = RetCalc.futureCapital(returns, 12, 0, 0, 500000)
      val futCapFix = RetCalc.futureCapital(FixedReturns(avg), 12, 0, 0, 500000)
      futCapVar should ===(futCapFix)
    }
  }

} 

package retcalc

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{Matchers, WordSpec}

class RetCalcIT extends WordSpec with Matchers with TypeCheckedTripleEquals {
  implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001)

  val params = RetCalcParams(
    nbOfMonthsInRetirement = 40 * 12,
    netIncome = 3000,
    currentExpenses = 2000,
    initialCapital = 10000)


  "simulate a retirement plan with real market data" in {
    val returns = Returns.fromEquityAndInflationData(
      equities = EquityData.fromResource("sp500.tsv"),
      inflations = InflationData.fromResource("cpi.tsv")).fromUntil("1952.09", "2017.10")

    val (capitalAtRetirement, capitalAfterDeath) =
      RetCalc.simulatePlan(returns, params = params, nbOfMonthsSavings = 25 * 12)
    capitalAtRetirement should ===(468924.5522)
    capitalAfterDeath should ===(2958841.7675)
  }
} 

package retcalc

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{EitherValues, Matchers, WordSpec}

class ReturnsSpec extends WordSpec with Matchers with TypeCheckedTripleEquals with EitherValues {

  implicit val doubleEquality: Equality[Double] =
    TolerantNumerics.tolerantDoubleEquality(0.0001)

  "Returns.monthlyReturn" should {
    "return a fixed rate for a FixedReturn" in {
      Returns.monthlyRate(FixedReturns(0.04), 0).right.value should ===(0.04 / 12)
      Returns.monthlyRate(FixedReturns(0.04), 10).right.value should ===(0.04 / 12)
    }

    val variableReturns = VariableReturns(Vector(
      VariableReturn("2000.01", 0.1),
      VariableReturn("2000.02", 0.2)))

    "return the nth rate for VariableReturn" in {
      Returns.monthlyRate(variableReturns, 0).right.value should ===(0.1)
      Returns.monthlyRate(variableReturns, 1).right.value should ===(0.2)
    }

    "return an error if n > length" in {
      Returns.monthlyRate(variableReturns, 2).left.value should ===(
        RetCalcError.ReturnMonthOutOfBounds(2, 1))
      Returns.monthlyRate(variableReturns, 3).left.value should ===(
        RetCalcError.ReturnMonthOutOfBounds(3, 1))
    }

    "return the n+offset th rate for OffsetReturn" in {
      val returns = OffsetReturns(variableReturns, 1)
      Returns.monthlyRate(returns, 0).right.value should ===(0.2)
    }
  }


  "Returns.fromEquityAndInflationData" should {
    "compute real total returns from equity and inflation data" in {
      val equities = Vector(
        EquityData("2117.01", 100.0, 10.0),
        EquityData("2117.02", 101.0, 12.0),
        EquityData("2117.03", 102.0, 12.0))

      val inflations = Vector(
        InflationData("2117.01", 100.0),
        InflationData("2117.02", 102.0),
        InflationData("2117.03", 102.0))

      val returns = Returns.fromEquityAndInflationData(equities, inflations)
      returns should ===(VariableReturns(Vector(
        VariableReturn("2117.02", (101.0 + 12.0 / 12) / 100.0 - 102.0 / 100.0),
        VariableReturn("2117.03", (102.0 + 12.0 / 12) / 101.0 - 102.0 / 102.0))))
    }
  }

  "VariableReturns.fromUntil" should {
    "keep only a window of the returns" in {
      val variableReturns = VariableReturns(Vector.tabulate(12) { i =>
        val d = (i + 1).toDouble
        VariableReturn(f"2017.$d%02.0f", d)
      })

      variableReturns.fromUntil("2017.07", "2017.09").returns should ===(Vector(
        VariableReturn("2017.07", 7.0),
        VariableReturn("2017.08", 8.0)
      ))

      variableReturns.fromUntil("2017.10", "2018.01").returns should ===(Vector(
        VariableReturn("2017.10", 10.0),
        VariableReturn("2017.11", 11.0),
        VariableReturn("2017.12", 12.0)
      ))
    }
  }

  "Returns.annualizedTotalReturn" should {
    val returns = VariableReturns(Vector.tabulate(12)(i => VariableReturn(i.toString, i.toDouble / 100 / 12)))
    val avg = Returns.annualizedTotalReturn(returns)
    "compute a geometric mean of the returns" in {
      // Excel: GEOMEAN (see geomean.ods)
      avg should ===(0.0549505735)
    }

    "compute an average that can be used to calculate a futureCapital instead of using variable returns" in {
      // This calculation only works if the capital does not change over time
      // otherwise, the capital fluctuates as well as the interest rates, and we cannot use the mean
      val futCapVar = RetCalc.futureCapital(returns, 12, 0, 0, 500000).right.value
      val futCapFix = RetCalc.futureCapital(FixedReturns(avg), 12, 0, 0, 500000).right.value
      futCapVar should ===(futCapFix)
    }
  }

} 

package retcalc

import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals}
import org.scalatest.{EitherValues, Matchers, WordSpec}

class RetCalcIT extends WordSpec with Matchers with TypeCheckedTripleEquals with EitherValues {
  implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001)

  val params = RetCalcParams(
    nbOfMonthsInRetirement = 40 * 12,
    netIncome = 3000,
    currentExpenses = 2000,
    initialCapital = 10000)


  "RetCalc.simulatePlan" should {
    "simulate a retirement plan with real market data" in {
      val returns = Returns.fromEquityAndInflationData(
        equities = EquityData.fromResource("sp500.tsv"),
        inflations = InflationData.fromResource("cpi.tsv")).fromUntil("1952.09", "2017.10")

      val (capitalAtRetirement, capitalAfterDeath) =
        RetCalc.simulatePlan(returns, params = params, nbOfMonthsSavings = 25 * 12).right.value
      capitalAtRetirement should ===(468924.5522)
      capitalAfterDeath should ===(2958841.7675)
    }
  }
}