Java Code Examples for org.nd4j.evaluation.classification.Evaluation#eval()

@Test
public void testEvaluation() {

    SparkDl4jMultiLayer sparkNet = getBasicNetwork();
    MultiLayerNetwork netCopy = sparkNet.getNetwork().clone();

    Evaluation evalExpected = new Evaluation();
    INDArray outLocal = netCopy.output(input, Layer.TrainingMode.TEST);
    evalExpected.eval(labels, outLocal);

    Evaluation evalActual = sparkNet.evaluate(sparkData);

    assertEquals(evalExpected.accuracy(), evalActual.accuracy(), 1e-3);
    assertEquals(evalExpected.f1(), evalActual.f1(), 1e-3);
    assertEquals(evalExpected.getNumRowCounter(), evalActual.getNumRowCounter(), 1e-3);
    assertMapEquals(evalExpected.falseNegatives(), evalActual.falseNegatives());
    assertMapEquals(evalExpected.falsePositives(), evalActual.falsePositives());
    assertMapEquals(evalExpected.trueNegatives(), evalActual.trueNegatives());
    assertMapEquals(evalExpected.truePositives(), evalActual.truePositives());
    assertEquals(evalExpected.precision(), evalActual.precision(), 1e-3);
    assertEquals(evalExpected.recall(), evalActual.recall(), 1e-3);
    assertEquals(evalExpected.getConfusionMatrix(), evalActual.getConfusionMatrix());
}
 

@Test
public void testStringHashLabels() {
    INDArray trueOutcome = FeatureUtil.toOutcomeVector(0, 2);
    INDArray predictedOutcome = FeatureUtil.toOutcomeVector(0, 2);

    Map<Integer, String> labelsMap = new HashMap<>();
    labelsMap.put(0, "hobbs");
    labelsMap.put(1, "cal");

    Evaluation eval = new Evaluation(labelsMap);

    eval.eval(trueOutcome, predictedOutcome);
    assertEquals(1, eval.classCount(0));
    assertEquals(labelsMap.get(0), eval.getClassLabel(0));

}
 

/**
 * Perform inference and then calculate the F1 score of the output(input) vs. the labels.
 *
 * @param input  the input to perform inference with
 * @param labels the true labels
 * @return the score for the given input,label pairs
 */
@Override
public double f1Score(INDArray input, INDArray labels) {
    feedForward(input);
    setLabels(labels);
    Evaluation eval = new Evaluation();
    eval.eval(labels, output(input));
    return eval.f1();
}
 

@Test
public void testFalsePerfectRecall() {
    int testSize = 100;
    int numClasses = 5;
    int winner = 1;
    int seed = 241;

    INDArray labels = Nd4j.zeros(testSize, numClasses);
    INDArray predicted = Nd4j.zeros(testSize, numClasses);

    Nd4j.getRandom().setSeed(seed);
    Random r = new Random(seed);

    //Modelling the situation when system predicts the same class every time
    for (int i = 0; i < testSize; i++) {
        //Generating random prediction but with a guaranteed winner
        INDArray rand = Nd4j.rand(1, numClasses);
        rand.put(0, winner, rand.sumNumber());
        rand.divi(rand.sumNumber());
        predicted.put(new INDArrayIndex[] {NDArrayIndex.point(i), all()}, rand);
        //Generating random label
        int label = r.nextInt(numClasses);
        labels.putScalar(new int[] {i, label}, 1.0);
    }

    //Explicitly specify the amount of classes
    Evaluation eval = new Evaluation(numClasses);
    eval.eval(labels, predicted);

    //For sure we shouldn't arrive at 100% recall unless we guessed everything right for every class
    assertNotEquals(1.0, eval.recall());
}
 

@Test
    public void testSingleClassBinaryClassification() {

        Evaluation eval = new Evaluation(1);

        for (int xe = 0; xe < 3; xe++) {
            INDArray zero = Nd4j.create(1,1);
            INDArray one = Nd4j.ones(1,1);

            //One incorrect, three correct
            eval.eval(one, zero);
            eval.eval(one, one);
            eval.eval(one, one);
            eval.eval(zero, zero);

//            System.out.println(eval.stats());
            eval.stats();

            assertEquals(0.75, eval.accuracy(), 1e-6);
            assertEquals(4, eval.getNumRowCounter());

            assertEquals(1, (int) eval.truePositives().get(0));
            assertEquals(2, (int) eval.truePositives().get(1));
            assertEquals(1, (int) eval.falseNegatives().get(1));

            eval.reset();
        }
    }
 

@Test
    public void testEvalInvalid() {
        Evaluation e = new Evaluation(5);
        e.eval(0, 1);
        e.eval(1, 0);
        e.eval(1, 1);

//        System.out.println(e.stats());
        e.stats();

        assertFalse(e.stats().contains("\uFFFD"));
    }
 

@Test
public void testBinaryCase() {
    INDArray ones10 = Nd4j.ones(10, 1);
    INDArray ones4 = Nd4j.ones(4, 1);
    INDArray zeros4 = Nd4j.zeros(4, 1);
    INDArray ones3 = Nd4j.ones(3, 1);
    INDArray zeros3 = Nd4j.zeros(3, 1);
    INDArray zeros2 = Nd4j.zeros(2, 1);

    Evaluation e = new Evaluation();
    e.eval(ones10, ones10); //10 true positives
    e.eval(ones3, zeros3); //3 false negatives
    e.eval(zeros4, ones4); //4 false positives
    e.eval(zeros2, zeros2); //2 true negatives


    assertEquals((10 + 2) / (double) (10 + 3 + 4 + 2), e.accuracy(), 1e-6);
    assertEquals(10, (int) e.truePositives().get(1));
    assertEquals(3, (int) e.falseNegatives().get(1));
    assertEquals(4, (int) e.falsePositives().get(1));
    assertEquals(2, (int) e.trueNegatives().get(1));

    //If we switch the label around: tp becomes tn, fp becomes fn, etc
    assertEquals(10, (int) e.trueNegatives().get(0));
    assertEquals(3, (int) e.falsePositives().get(0));
    assertEquals(4, (int) e.falseNegatives().get(0));
    assertEquals(2, (int) e.truePositives().get(0));
}
 

@Test
    public void testLabelReset(){

        Map<Integer,String> m = new HashMap<>();
        m.put(0, "False");
        m.put(1, "True");

        Evaluation e1 = new Evaluation(m);
        INDArray zero = Nd4j.create(new double[]{1,0}).reshape(1,2);
        INDArray one = Nd4j.create(new double[]{0,1}).reshape(1,2);

        e1.eval(zero, zero);
        e1.eval(zero, zero);
        e1.eval(one, zero);
        e1.eval(one, one);
        e1.eval(one, one);
        e1.eval(one, one);

        String s1 = e1.stats();
//        System.out.println(s1);

        e1.reset();
        e1.eval(zero, zero);
        e1.eval(zero, zero);
        e1.eval(one, zero);
        e1.eval(one, one);
        e1.eval(one, one);
        e1.eval(one, one);

        String s2 = e1.stats();
        assertEquals(s1, s2);
    }
 

@Test
public void testEvaluationNaNs(){

    Evaluation e = new Evaluation();
    INDArray predictions = Nd4j.create(new double[]{0.1, Double.NaN, 0.3}, new long[]{1,3});
    INDArray labels = Nd4j.create(new double[]{0, 0, 1}, new long[]{1,3});

    try {
        e.eval(labels, predictions);
    } catch (IllegalStateException ex){
        assertTrue(ex.getMessage().contains("NaN"));
    }

}
 

@Test
public void testNewInstances() {
    boolean print = true;
    Nd4j.getRandom().setSeed(12345);

    Evaluation evaluation = new Evaluation();
    EvaluationBinary evaluationBinary = new EvaluationBinary();
    ROC roc = new ROC(2);
    ROCBinary roc2 = new ROCBinary(2);
    ROCMultiClass roc3 = new ROCMultiClass(2);
    RegressionEvaluation regressionEvaluation = new RegressionEvaluation();
    EvaluationCalibration ec = new EvaluationCalibration();


    IEvaluation[] arr = new IEvaluation[] {evaluation, evaluationBinary, roc, roc2, roc3, regressionEvaluation, ec};

    INDArray evalLabel1 = Nd4j.create(10, 3);
    for (int i = 0; i < 10; i++) {
        evalLabel1.putScalar(i, i % 3, 1.0);
    }
    INDArray evalProb1 = Nd4j.rand(10, 3);
    evalProb1.diviColumnVector(evalProb1.sum(1));

    evaluation.eval(evalLabel1, evalProb1);
    roc3.eval(evalLabel1, evalProb1);
    ec.eval(evalLabel1, evalProb1);

    INDArray evalLabel2 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 3), 0.5));
    INDArray evalProb2 = Nd4j.rand(10, 3);
    evaluationBinary.eval(evalLabel2, evalProb2);
    roc2.eval(evalLabel2, evalProb2);

    INDArray evalLabel3 = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(10, 1), 0.5));
    INDArray evalProb3 = Nd4j.rand(10, 1);
    roc.eval(evalLabel3, evalProb3);

    INDArray reg1 = Nd4j.rand(10, 3);
    INDArray reg2 = Nd4j.rand(10, 3);

    regressionEvaluation.eval(reg1, reg2);

    Evaluation evaluation2 = evaluation.newInstance();
    EvaluationBinary evaluationBinary2 = evaluationBinary.newInstance();
    ROC roc_2 = roc.newInstance();
    ROCBinary roc22 = roc2.newInstance();
    ROCMultiClass roc32 = roc3.newInstance();
    RegressionEvaluation regressionEvaluation2 = regressionEvaluation.newInstance();
    EvaluationCalibration ec2 = ec.newInstance();

    IEvaluation[] arr2 = new IEvaluation[] {evaluation2, evaluationBinary2, roc_2, roc22, roc32, regressionEvaluation2, ec2};

    evaluation2.eval(evalLabel1, evalProb1);
    roc32.eval(evalLabel1, evalProb1);
    ec2.eval(evalLabel1, evalProb1);

    evaluationBinary2.eval(evalLabel2, evalProb2);
    roc22.eval(evalLabel2, evalProb2);

    roc_2.eval(evalLabel3, evalProb3);

    regressionEvaluation2.eval(reg1, reg2);

    for (int i = 0 ; i < arr.length ; i++) {

        IEvaluation e = arr[i];
        IEvaluation e2 = arr2[i];
        assertEquals("Json not equal ", e.toJson(), e2.toJson());
        assertEquals("Stats not equal ", e.stats(), e2.stats());
    }
}
 

public static void main(String[] args) throws Exception {

        // First: get the dataset using the record reader. CSVRecordReader handles
        // loading/parsing
        int numLinesToSkip = 0;
        char delimiter = ',';
        RecordReader recordReader = new CSVRecordReader(numLinesToSkip, delimiter);
        recordReader.initialize(new FileSplit(new ClassPathResource("DL4J_Resources/iris.txt").getFile()));

        // Second: the RecordReaderDataSetIterator handles conversion to DataSet
        // objects, ready for use in neural network
        int labelIndex = 4; // 5 values in each row of the iris.txt CSV: 4 input features followed by an
                            // integer label (class) index. Labels are the 5th value (index 4) in each row
        int numClasses = 3; // 3 classes (types of iris flowers) in the iris data set. Classes have integer
                            // values 0, 1 or 2
        int batchSize = 150; // Iris data set: 150 examples total. We are loading all of them into one
                             // DataSet (not recommended for large data sets)

        DataSetIterator iterator = new RecordReaderDataSetIterator(recordReader, batchSize, labelIndex, numClasses);
        DataSet allData = iterator.next();
        allData.shuffle();
        SplitTestAndTrain testAndTrain = allData.splitTestAndTrain(0.65); // Use 65% of data for training

        DataSet trainingData = testAndTrain.getTrain();
        DataSet testData = testAndTrain.getTest();

        // We need to normalize our data. We'll use NormalizeStandardize (which gives us
        // mean 0, unit variance):
        DataNormalization normalizer = new NormalizerStandardize();
        normalizer.fit(trainingData); // Collect the statistics (mean/stdev) from the training data. This does not
                                      // modify the input data
        normalizer.transform(trainingData); // Apply normalization to the training data
        normalizer.transform(testData); // Apply normalization to the test data. This is using statistics calculated
                                        // from the *training* set

        final int numInputs = 4;
        int outputNum = 3;
        long seed = 6;

        log.info("Build model....");
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed).activation(Activation.TANH)
                .weightInit(WeightInit.XAVIER).updater(new Sgd(0.1)).l2(1e-4).list()
                .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(3).build())
                .layer(1, new DenseLayer.Builder().nIn(3).nOut(3).build())
                .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                        .activation(Activation.SOFTMAX).nIn(3).nOut(outputNum).build())
                .build();

        // run the model
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();
        model.setListeners(new ScoreIterationListener(100));

        for (int i = 0; i < 1000; i++) {
            model.fit(trainingData);
        }

        // evaluate the model on the test set
        Evaluation eval = new Evaluation(3);
        INDArray input = testData.getFeatures();
        INDArray output = model.output(input);
        System.out.println("INPUT:" + input.toString());
        eval.eval(testData.getLabels(), output);
        log.info(eval.stats());

        // Save the model
        File locationToSave = new File("src/main/resources/generatedModels/DL4J/DL4J_Iris_Model.zip"); // Where to save
        // the network.
        // Note: the file
        // is in .zip
        // format - can
        // be opened
        // externally
        boolean saveUpdater = true; // Updater: i.e., the state for Momentum, RMSProp, Adagrad etc. Save this if you
        // want to train your network more in the future
        // ModelSerializer.writeModel(model, locationToSave, saveUpdater);

        // Load the model
        MultiLayerNetwork restored = ModelSerializer.restoreMultiLayerNetwork(locationToSave);

        System.out.println("Saved and loaded parameters are equal:      " + model.params().equals(restored.params()));
        System.out.println("Saved and loaded configurations are equal:  "
                + model.getLayerWiseConfigurations().equals(restored.getLayerWiseConfigurations()));

    }
 

@Test
public void testEvaluationCustomBinaryThreshold() {
    Nd4j.getRandom().setSeed(12345);

    //Sanity checks: 0.5 threshold for 1-output and 2-output binary cases
    Evaluation e = new Evaluation();
    Evaluation e05 = new Evaluation(0.5);
    Evaluation e05v2 = new Evaluation(0.5);

    int nExamples = 20;
    int nOut = 2;
    INDArray probs = Nd4j.rand(nExamples, nOut);
    probs.diviColumnVector(probs.sum(1));
    INDArray labels = Nd4j.create(nExamples, nOut);
    Random r = new Random(12345);
    for (int i = 0; i < nExamples; i++) {
        labels.putScalar(i, r.nextInt(2), 1.0);
    }

    e.eval(labels, probs);
    e05.eval(labels, probs);
    e05v2.eval(labels.getColumn(1, true), probs.getColumn(1, true)); //"single output binary" case

    for (Evaluation e2 : new Evaluation[] {e05, e05v2}) {
        assertEquals(e.accuracy(), e2.accuracy(), 1e-6);
        assertEquals(e.f1(), e2.f1(), 1e-6);
        assertEquals(e.precision(), e2.precision(), 1e-6);
        assertEquals(e.recall(), e2.recall(), 1e-6);
        assertEquals(e.getConfusionMatrix(), e2.getConfusionMatrix());
    }

    //Check with decision threshold of 0.25
    //In this test, we'll cheat a bit: multiply class 1 probabilities by 2 (max of 1.0); this should give an
    // identical result to a threshold of 0.5 vs. no multiplication and threshold of 0.25

    INDArray p2 = probs.dup();
    INDArray p2c = p2.getColumn(1);
    p2c.muli(2.0);
    Nd4j.getExecutioner().exec(new ScalarMin(p2c, null, p2c, 1.0));
    p2.getColumn(0).assign(p2.getColumn(1).rsub(1.0));

    Evaluation e025 = new Evaluation(0.25);
    e025.eval(labels, probs);

    Evaluation ex2 = new Evaluation();
    ex2.eval(labels, p2);

    assertEquals(ex2.accuracy(), e025.accuracy(), 1e-6);
    assertEquals(ex2.f1(), e025.f1(), 1e-6);
    assertEquals(ex2.precision(), e025.precision(), 1e-6);
    assertEquals(ex2.recall(), e025.recall(), 1e-6);
    assertEquals(ex2.getConfusionMatrix(), e025.getConfusionMatrix());


    //Check the same thing, but the single binary output case:

    Evaluation e025v2 = new Evaluation(0.25);
    e025v2.eval(labels.getColumn(1, true), probs.getColumn(1, true));

    assertEquals(ex2.accuracy(), e025v2.accuracy(), 1e-6);
    assertEquals(ex2.f1(), e025v2.f1(), 1e-6);
    assertEquals(ex2.precision(), e025v2.precision(), 1e-6);
    assertEquals(ex2.recall(), e025v2.recall(), 1e-6);
    assertEquals(ex2.getConfusionMatrix(), e025v2.getConfusionMatrix());
}
 

@Test
public void testEvaluationBinary() {
    //Compare EvaluationBinary to Evaluation class
    DataType dtypeBefore = Nd4j.defaultFloatingPointType();
    EvaluationBinary first = null;
    String sFirst = null;
    try {
        for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.INT}) {
            Nd4j.setDefaultDataTypes(globalDtype, globalDtype.isFPType() ? globalDtype : DataType.DOUBLE);
            for (DataType lpDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {

                Nd4j.getRandom().setSeed(12345);

                int nExamples = 50;
                int nOut = 4;
                long[] shape = {nExamples, nOut};

                INDArray labels = Nd4j.getExecutioner().exec(new BernoulliDistribution(Nd4j.createUninitialized(lpDtype, shape), 0.5));

                INDArray predicted = Nd4j.rand(lpDtype, shape);
                INDArray binaryPredicted = predicted.gt(0.5);

                EvaluationBinary eb = new EvaluationBinary();
                eb.eval(labels, predicted);

                //System.out.println(eb.stats());

                double eps = 1e-6;
                for (int i = 0; i < nOut; i++) {
                    INDArray lCol = labels.getColumn(i,true);
                    INDArray pCol = predicted.getColumn(i,true);
                    INDArray bpCol = binaryPredicted.getColumn(i,true);

                    int countCorrect = 0;
                    int tpCount = 0;
                    int tnCount = 0;
                    for (int j = 0; j < lCol.length(); j++) {
                        if (lCol.getDouble(j) == bpCol.getDouble(j)) {
                            countCorrect++;
                            if (lCol.getDouble(j) == 1) {
                                tpCount++;
                            } else {
                                tnCount++;
                            }
                        }
                    }
                    double acc = countCorrect / (double) lCol.length();

                    Evaluation e = new Evaluation();
                    e.eval(lCol, pCol);

                    assertEquals(acc, eb.accuracy(i), eps);
                    assertEquals(e.accuracy(), eb.scoreForMetric(ACCURACY, i), eps);
                    assertEquals(e.precision(1), eb.scoreForMetric(PRECISION, i), eps);
                    assertEquals(e.recall(1), eb.scoreForMetric(RECALL, i), eps);
                    assertEquals(e.f1(1), eb.scoreForMetric(F1, i), eps);
                    assertEquals(e.falseAlarmRate(), eb.scoreForMetric(FAR, i), eps);
                    assertEquals(e.falsePositiveRate(1), eb.falsePositiveRate(i), eps);


                    assertEquals(tpCount, eb.truePositives(i));
                    assertEquals(tnCount, eb.trueNegatives(i));

                    assertEquals((int) e.truePositives().get(1), eb.truePositives(i));
                    assertEquals((int) e.trueNegatives().get(1), eb.trueNegatives(i));
                    assertEquals((int) e.falsePositives().get(1), eb.falsePositives(i));
                    assertEquals((int) e.falseNegatives().get(1), eb.falseNegatives(i));

                    assertEquals(nExamples, eb.totalCount(i));

                    String s = eb.stats();
                    if(first == null) {
                        first = eb;
                        sFirst = s;
                    } else {
                        assertEquals(first, eb);
                        assertEquals(sFirst, s);
                    }
                }
            }
        }
    } finally {
        Nd4j.setDefaultDataTypes(dtypeBefore, dtypeBefore);
    }
}
 

@Test
public void testEvalStatsBinaryCase(){
    //Make sure we report class 1 precision/recall/f1 not macro averaged, for binary case

    Evaluation e = new Evaluation();

    INDArray l0 = Nd4j.createFromArray(new double[]{1,0}).reshape(1,2);
    INDArray l1 = Nd4j.createFromArray(new double[]{0,1}).reshape(1,2);

    e.eval(l1, l1);
    e.eval(l1, l1);
    e.eval(l1, l1);
    e.eval(l0, l0);
    e.eval(l1, l0);
    e.eval(l1, l0);
    e.eval(l0, l1);

    double tp = 3;
    double fp = 1;
    double fn = 2;

    double prec = tp / (tp + fp);
    double rec = tp / (tp + fn);
    double f1 = 2 * prec * rec / (prec + rec);

    assertEquals(prec, e.precision(), 1e-6);
    assertEquals(rec, e.recall(), 1e-6);

    DecimalFormat df = new DecimalFormat("0.0000");

    String stats = e.stats();
    //System.out.println(stats);

    String stats2 = stats.replaceAll("( )+", " ");

    String recS = " Recall: " + df.format(rec);
    String preS = " Precision: " + df.format(prec);
    String f1S = "F1 Score: " + df.format(f1);

    assertTrue(stats2, stats2.contains(recS));
    assertTrue(stats2, stats2.contains(preS));
    assertTrue(stats2, stats2.contains(f1S));
}
 

@Test
public void testSegmentation(){
    for( int c : new int[]{4, 1}) { //c=1 should be treated as binary classification case
        Nd4j.getRandom().setSeed(12345);
        int mb = 3;
        int h = 3;
        int w = 2;

        //NCHW
        INDArray labels = Nd4j.create(DataType.FLOAT, mb, c, h, w);
        Random r = new Random(12345);
        for (int i = 0; i < mb; i++) {
            for (int j = 0; j < h; j++) {
                for (int k = 0; k < w; k++) {
                    if(c == 1){
                        labels.putScalar(i, 0, j, k, r.nextInt(2));
                    } else {
                        int classIdx = r.nextInt(c);
                        labels.putScalar(i, classIdx, j, k, 1.0);
                    }
                }
            }
        }

        INDArray predictions = Nd4j.rand(DataType.FLOAT, mb, c, h, w);
        if(c > 1) {
            DynamicCustomOp op = DynamicCustomOp.builder("softmax")
                    .addInputs(predictions)
                    .addOutputs(predictions)
                    .callInplace(true)
                    .addIntegerArguments(1) //Axis
                    .build();
            Nd4j.exec(op);
        }

        Evaluation e2d = new Evaluation();
        Evaluation e4d = new Evaluation();

        e4d.eval(labels, predictions);

        for (int i = 0; i < mb; i++) {
            for (int j = 0; j < h; j++) {
                for (int k = 0; k < w; k++) {
                    INDArray rowLabel = labels.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j), NDArrayIndex.point(k));
                    INDArray rowPredictions = predictions.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(j), NDArrayIndex.point(k));
                    rowLabel = rowLabel.reshape(1, rowLabel.length());
                    rowPredictions = rowPredictions.reshape(1, rowLabel.length());

                    e2d.eval(rowLabel, rowPredictions);
                }
            }
        }

        assertEquals(e2d, e4d);


        //NHWC, etc
        INDArray lOrig = labels;
        INDArray fOrig = predictions;
        for (int i = 0; i < 4; i++) {
            switch (i) {
                case 0:
                    //CNHW - Never really used
                    labels = lOrig.permute(1, 0, 2, 3).dup();
                    predictions = fOrig.permute(1, 0, 2, 3).dup();
                    break;
                case 1:
                    //NCHW
                    labels = lOrig;
                    predictions = fOrig;
                    break;
                case 2:
                    //NHCW - Never really used...
                    labels = lOrig.permute(0, 2, 1, 3).dup();
                    predictions = fOrig.permute(0, 2, 1, 3).dup();
                    break;
                case 3:
                    //NHWC
                    labels = lOrig.permute(0, 2, 3, 1).dup();
                    predictions = fOrig.permute(0, 2, 3, 1).dup();
                    break;
                default:
                    throw new RuntimeException();
            }

            Evaluation e = new Evaluation();
            e.setAxis(i);

            e.eval(labels, predictions);
            assertEquals(e2d, e);
        }
    }
}
 

private static void apply(Evaluation e, int nTimes, INDArray predicted, INDArray actual) {
    for (int i = 0; i < nTimes; i++) {
        e.eval(actual, predicted);
    }
}
 

@Test
public void testTopNAccuracyMerging() {

    Evaluation e1 = new Evaluation(null, 3);
    Evaluation e2 = new Evaluation(null, 3);

    INDArray i0 = Nd4j.create(new double[] {1, 0, 0, 0, 0}, new long[]{1, 5});
    INDArray i1 = Nd4j.create(new double[] {0, 1, 0, 0, 0}, new long[]{1, 5});

    INDArray p0_0 = Nd4j.create(new double[] {0.8, 0.05, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 0: highest prob
    INDArray p0_1 = Nd4j.create(new double[] {0.4, 0.45, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 0: 2nd highest prob
    INDArray p0_2 = Nd4j.create(new double[] {0.1, 0.45, 0.35, 0.05, 0.05}, new long[]{1, 5}); //class 0: 3rd highest prob
    INDArray p0_3 = Nd4j.create(new double[] {0.1, 0.40, 0.30, 0.15, 0.05}, new long[]{1, 5}); //class 0: 4th highest prob

    INDArray p1_0 = Nd4j.create(new double[] {0.05, 0.80, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 1: highest prob
    INDArray p1_1 = Nd4j.create(new double[] {0.45, 0.40, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 1: 2nd highest prob
    INDArray p1_2 = Nd4j.create(new double[] {0.35, 0.10, 0.45, 0.05, 0.05}, new long[]{1, 5}); //class 1: 3rd highest prob
    INDArray p1_3 = Nd4j.create(new double[] {0.40, 0.10, 0.30, 0.15, 0.05}, new long[]{1, 5}); //class 1: 4th highest prob


    //                                              Correct     TopNCorrect     Total
    e1.eval(i0, p0_0); //  1           1               1
    e1.eval(i0, p0_1); //  1           2               2
    e1.eval(i0, p0_2); //  1           3               3
    e1.eval(i0, p0_3); //  1           3               4
    assertEquals(0.25, e1.accuracy(), 1e-6);
    assertEquals(0.75, e1.topNAccuracy(), 1e-6);
    assertEquals(3, e1.getTopNCorrectCount());
    assertEquals(4, e1.getTopNTotalCount());

    e2.eval(i1, p1_0); //  1           1               1
    e2.eval(i1, p1_1); //  1           2               2
    e2.eval(i1, p1_2); //  1           3               3
    e2.eval(i1, p1_3); //  1           3               4
    assertEquals(1.0 / 4, e2.accuracy(), 1e-6);
    assertEquals(3.0 / 4, e2.topNAccuracy(), 1e-6);
    assertEquals(3, e2.getTopNCorrectCount());
    assertEquals(4, e2.getTopNTotalCount());

    e1.merge(e2);

    assertEquals(8, e1.getNumRowCounter());
    assertEquals(8, e1.getTopNTotalCount());
    assertEquals(6, e1.getTopNCorrectCount());
    assertEquals(2.0 / 8, e1.accuracy(), 1e-6);
    assertEquals(6.0 / 8, e1.topNAccuracy(), 1e-6);
}
 

@Test
    public void testTopNAccuracy() {

        Evaluation e = new Evaluation(null, 3);

        INDArray i0 = Nd4j.create(new double[] {1, 0, 0, 0, 0}, new long[]{1, 5});
        INDArray i1 = Nd4j.create(new double[] {0, 1, 0, 0, 0}, new long[]{1, 5});

        INDArray p0_0 = Nd4j.create(new double[] {0.8, 0.05, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 0: highest prob
        INDArray p0_1 = Nd4j.create(new double[] {0.4, 0.45, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 0: 2nd highest prob
        INDArray p0_2 = Nd4j.create(new double[] {0.1, 0.45, 0.35, 0.05, 0.05}, new long[]{1, 5}); //class 0: 3rd highest prob
        INDArray p0_3 = Nd4j.create(new double[] {0.1, 0.40, 0.30, 0.15, 0.05}, new long[]{1, 5}); //class 0: 4th highest prob

        INDArray p1_0 = Nd4j.create(new double[] {0.05, 0.80, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 1: highest prob
        INDArray p1_1 = Nd4j.create(new double[] {0.45, 0.40, 0.05, 0.05, 0.05}, new long[]{1, 5}); //class 1: 2nd highest prob
        INDArray p1_2 = Nd4j.create(new double[] {0.35, 0.10, 0.45, 0.05, 0.05}, new long[]{1, 5}); //class 1: 3rd highest prob
        INDArray p1_3 = Nd4j.create(new double[] {0.40, 0.10, 0.30, 0.15, 0.05}, new long[]{1, 5}); //class 1: 4th highest prob


        //                                              Correct     TopNCorrect     Total
        e.eval(i0, p0_0); //  1           1               1
        assertEquals(1.0, e.accuracy(), 1e-6);
        assertEquals(1.0, e.topNAccuracy(), 1e-6);
        assertEquals(1, e.getTopNCorrectCount());
        assertEquals(1, e.getTopNTotalCount());
        e.eval(i0, p0_1); //  1           2               2
        assertEquals(0.5, e.accuracy(), 1e-6);
        assertEquals(1.0, e.topNAccuracy(), 1e-6);
        assertEquals(2, e.getTopNCorrectCount());
        assertEquals(2, e.getTopNTotalCount());
        e.eval(i0, p0_2); //  1           3               3
        assertEquals(1.0 / 3, e.accuracy(), 1e-6);
        assertEquals(1.0, e.topNAccuracy(), 1e-6);
        assertEquals(3, e.getTopNCorrectCount());
        assertEquals(3, e.getTopNTotalCount());
        e.eval(i0, p0_3); //  1           3               4
        assertEquals(0.25, e.accuracy(), 1e-6);
        assertEquals(0.75, e.topNAccuracy(), 1e-6);
        assertEquals(3, e.getTopNCorrectCount());
        assertEquals(4, e.getTopNTotalCount());

        e.eval(i1, p1_0); //  2           4               5
        assertEquals(2.0 / 5, e.accuracy(), 1e-6);
        assertEquals(4.0 / 5, e.topNAccuracy(), 1e-6);
        e.eval(i1, p1_1); //  2           5               6
        assertEquals(2.0 / 6, e.accuracy(), 1e-6);
        assertEquals(5.0 / 6, e.topNAccuracy(), 1e-6);
        e.eval(i1, p1_2); //  2           6               7
        assertEquals(2.0 / 7, e.accuracy(), 1e-6);
        assertEquals(6.0 / 7, e.topNAccuracy(), 1e-6);
        e.eval(i1, p1_3); //  2           6               8
        assertEquals(2.0 / 8, e.accuracy(), 1e-6);
        assertEquals(6.0 / 8, e.topNAccuracy(), 1e-6);
        assertEquals(6, e.getTopNCorrectCount());
        assertEquals(8, e.getTopNTotalCount());

//        System.out.println(e.stats());
        e.stats();
    }
 

@Test
    public void testEvalMethods() {
        //Check eval(int,int) vs. eval(INDArray,INDArray)

        Evaluation e1 = new Evaluation(4);
        Evaluation e2 = new Evaluation(4);

        INDArray i0 = Nd4j.create(new double[] {1, 0, 0, 0}, new long[]{1, 4});
        INDArray i1 = Nd4j.create(new double[] {0, 1, 0, 0}, new long[]{1, 4});
        INDArray i2 = Nd4j.create(new double[] {0, 0, 1, 0}, new long[]{1, 4});
        INDArray i3 = Nd4j.create(new double[] {0, 0, 0, 1}, new long[]{1, 4});

        e1.eval(i0, i0); //order: actual, predicted
        e2.eval(0, 0); //order: predicted, actual
        e1.eval(i0, i2);
        e2.eval(2, 0);
        e1.eval(i0, i2);
        e2.eval(2, 0);
        e1.eval(i1, i2);
        e2.eval(2, 1);
        e1.eval(i3, i3);
        e2.eval(3, 3);
        e1.eval(i3, i0);
        e2.eval(0, 3);
        e1.eval(i3, i0);
        e2.eval(0, 3);

        org.nd4j.evaluation.classification.ConfusionMatrix<Integer> cm = e1.getConfusionMatrix();
        assertEquals(1, cm.getCount(0, 0)); //Order: actual, predicted
        assertEquals(2, cm.getCount(0, 2));
        assertEquals(1, cm.getCount(1, 2));
        assertEquals(1, cm.getCount(3, 3));
        assertEquals(2, cm.getCount(3, 0));

//        System.out.println(e1.stats());
//        System.out.println(e2.stats());
        e1.stats();
        e2.stats();

        assertEquals(e1.stats(), e2.stats());
    }
 

@Test
public void testEval2() {

    DataType dtypeBefore = Nd4j.defaultFloatingPointType();
    Evaluation first = null;
    String sFirst = null;
    try {
        for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.INT}) {
            Nd4j.setDefaultDataTypes(globalDtype, globalDtype.isFPType() ? globalDtype : DataType.DOUBLE);
            for (DataType lpDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {

                //Confusion matrix:
                //actual 0      20      3
                //actual 1      10      5

                Evaluation evaluation = new Evaluation(Arrays.asList("class0", "class1"));
                INDArray predicted0 = Nd4j.create(new double[]{1, 0}, new long[]{1, 2}).castTo(lpDtype);
                INDArray predicted1 = Nd4j.create(new double[]{0, 1}, new long[]{1, 2}).castTo(lpDtype);
                INDArray actual0 = Nd4j.create(new double[]{1, 0}, new long[]{1, 2}).castTo(lpDtype);
                INDArray actual1 = Nd4j.create(new double[]{0, 1}, new long[]{1, 2}).castTo(lpDtype);
                for (int i = 0; i < 20; i++) {
                    evaluation.eval(actual0, predicted0);
                }

                for (int i = 0; i < 3; i++) {
                    evaluation.eval(actual0, predicted1);
                }

                for (int i = 0; i < 10; i++) {
                    evaluation.eval(actual1, predicted0);
                }

                for (int i = 0; i < 5; i++) {
                    evaluation.eval(actual1, predicted1);
                }

                assertEquals(20, evaluation.truePositives().get(0), 0);
                assertEquals(3, evaluation.falseNegatives().get(0), 0);
                assertEquals(10, evaluation.falsePositives().get(0), 0);
                assertEquals(5, evaluation.trueNegatives().get(0), 0);

                assertEquals((20.0 + 5) / (20 + 3 + 10 + 5), evaluation.accuracy(), 1e-6);

                String s = evaluation.stats();

                if(first == null) {
                    first = evaluation;
                    sFirst = s;
                } else {
                    assertEquals(first, evaluation);
                    assertEquals(sFirst, s);
                }
            }
        }
    } finally {
        Nd4j.setDefaultDataTypes(dtypeBefore, dtypeBefore);
    }
}