Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#mul()

@Test
public void testExecution(){
    List<PythonVariable> inputs = new ArrayList<>();
    INDArray x = Nd4j.ones(dataType, shape);
    INDArray y = Nd4j.zeros(dataType, shape);
    INDArray z = (dataType == DataType.BOOL)?x:x.mul(y.add(2));
    z = (dataType == DataType.BFLOAT16)? z.castTo(DataType.FLOAT): z;
    PythonType<INDArray> arrType = PythonTypes.get("numpy.ndarray");
    inputs.add(new PythonVariable<>("x", arrType, x));
    inputs.add(new PythonVariable<>("y", arrType, y));
    List<PythonVariable> outputs = new ArrayList<>();
    PythonVariable<INDArray> output = new PythonVariable<>("z", arrType);
    outputs.add(output);
    String code = (dataType == DataType.BOOL)?"z = x":"z = x * (y + 2)";
    if (shape.length == 0){ // scalar special case
        code += "\nimport numpy as np\nz = np.asarray(float(z), dtype=x.dtype)";
    }
    PythonExecutioner.exec(code, inputs, outputs);
    INDArray z2 = output.getValue();

    Assert.assertEquals(z.dataType(), z2.dataType());
    Assert.assertEquals(z, z2);

}
 

@Test
public void testEvalAdd() {
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Nd4j.linspace(1, 4, 4);
    INDArray yArr = arr.dup();
    SDVariable x = sameDiff.var("x", arr);
    SDVariable y = sameDiff.var("y", yArr);

    SDVariable sigmoid = x.mul(y);
    INDArray assertion = arr.mul(arr);
    Map<String, INDArray> vars = new HashMap<>();
    vars.put("x", arr);
    vars.put("y", yArr);
    INDArray eval = sameDiff.output(vars, Collections.singletonList(sigmoid.name())).get(sigmoid.name());
    assertEquals(assertion, eval);
}
 

@Before
public void setUp() {
    SUT = new MultiNormalizerMinMaxScaler();
    SUT.fitLabel(true);

    // Prepare test data
    int nSamples = 5120;

    INDArray values = Nd4j.linspace(1, nSamples, nSamples, Nd4j.dataType()).reshape(1, -1).transpose();
    INDArray input1 = values.mul(INPUT1_SCALE);
    INDArray input2 = values.mul(INPUT2_SCALE);
    INDArray output1 = values.mul(OUTPUT1_SCALE);
    INDArray output2 = values.mul(OUTPUT2_SCALE);

    data = new MultiDataSet(new INDArray[] {input1, input2}, new INDArray[] {output1, output2});

    naturalMin = 1;
    naturalMax = nSamples;
}
 

@Test
    public void testSimple(){
        Nd4j.create(1);
        for(DataType dt : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.INT, DataType.LONG}) {
//            System.out.println("----- " + dt + " -----");
            INDArray arr = Nd4j.ones(dt,1, 5);
//            System.out.println("Ones: " + arr);
            arr.assign(1.0);
//            System.out.println("assign(1.0): " + arr);
//            System.out.println("DIV: " + arr.div(8));
//            System.out.println("MUL: " + arr.mul(8));
//            System.out.println("SUB: " + arr.sub(8));
//            System.out.println("ADD: " + arr.add(8));
//            System.out.println("RDIV: " + arr.rdiv(8));
//            System.out.println("RSUB: " + arr.rsub(8));
            arr.div(8);
            arr.mul(8);
            arr.sub(8);
            arr.add(8);
            arr.rdiv(8);
            arr.rsub(8);
        }
    }
 

@Before
public void setUp() {
    SUT = new MultiNormalizerStandardize();
    SUT.fitLabel(true);

    // Prepare test data
    int nSamples = 5120;

    INDArray values = Nd4j.linspace(1, nSamples, nSamples, Nd4j.dataType()).reshape(1, -1).transpose();
    INDArray input1 = values.mul(INPUT1_SCALE);
    INDArray input2 = values.mul(INPUT2_SCALE);
    INDArray output1 = values.mul(OUTPUT1_SCALE);
    INDArray output2 = values.mul(OUTPUT2_SCALE);

    data = new MultiDataSet(new INDArray[] {input1, input2}, new INDArray[] {output1, output2});

    meanNaturalNums = (nSamples + 1) / 2.0;
    stdNaturalNums = Math.sqrt((nSamples * nSamples - 1) / 12.0);
}
 

@Override
public INDArray getActivation(INDArray in, boolean training) {
    if (training) {
        try(MemoryWorkspace ignored = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
            this.alpha = Nd4j.rand(l, u, Nd4j.getRandom(), in.shape());
        }
        INDArray inTimesAlpha = in.mul(alpha);
        BooleanIndexing.replaceWhere(in, inTimesAlpha, Conditions.lessThan(0));
    } else {
        this.alpha = null;
        double a = 0.5 * (l + u);
        return Nd4j.getExecutioner().exec(new RectifiedLinear(in, a));
    }

    return in;
}
 

@Test
public void testDebugEdgeCase2(){
    DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
    INDArray l1 = Nd4j.create(new double[]{-0.2585039112684677,-0.005179485353710878,0.4348343401770497,0.020356532375728764,-0.1970793298488186});
    INDArray l2 = Nd4j.create(2,l1.size(1));

    INDArray p1 = Nd4j.create(new double[]{1.3979850406519119,0.6169451410155852,1.128993957530918,0.21000426084450596,0.3171215178932696});
    INDArray p2 = Nd4j.create(2, p1.size(1));

    for( int i=0; i<2; i++ ){
        l2.putRow(i, l1);
        p2.putRow(i, p1);
    }

    INDArray norm2_1 = l1.norm2(1);
    INDArray temp1 = p1.mul(l1);
    INDArray out1 = temp1.diviColumnVector(norm2_1);

    INDArray norm2_2 = l2.norm2(1);
    INDArray temp2 = p2.mul(l2);
    INDArray out2 = temp2.diviColumnVector(norm2_2);

    System.out.println("norm2_1: " + Arrays.toString(norm2_1.data().asDouble()));
    System.out.println("norm2_2: " + Arrays.toString(norm2_2.data().asDouble()));

    System.out.println("temp1: " + Arrays.toString(temp1.data().asDouble()));
    System.out.println("temp2: " + Arrays.toString(temp2.data().asDouble()));

    //Outputs here should be identical:
    System.out.println(Arrays.toString(out1.data().asDouble()));
    System.out.println(Arrays.toString(out2.getRow(0).dup().data().asDouble()));
}
 

private INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask){
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype

    INDArray output = activationFn.getActivation(preOutput.dup(), true);

    INDArray scoreArr = labels.mul(output);
    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

@Test
public void testInplaceExecution(){
    if (dataType == DataType.BOOL || dataType == DataType.BFLOAT16)return;
    if (shape.length == 0) return;
    List<PythonVariable> inputs = new ArrayList<>();
    INDArray x = Nd4j.ones(dataType, shape);
    INDArray y = Nd4j.zeros(dataType, shape);
    INDArray z = x.mul(y.add(2));
   // Nd4j.getAffinityManager().ensureLocation(z, AffinityManager.Location.HOST);
    PythonType<INDArray> arrType = PythonTypes.get("numpy.ndarray");
    inputs.add(new PythonVariable<>("x", arrType, x));
    inputs.add(new PythonVariable<>("y", arrType, y));
    List<PythonVariable> outputs = new ArrayList<>();
    PythonVariable<INDArray> output = new PythonVariable<>("x", arrType);
    outputs.add(output);
    String code = "x *= y + 2";
    PythonExecutioner.exec(code, inputs, outputs);
    INDArray z2 = output.getValue();
    Assert.assertEquals(x.dataType(), z2.dataType());
    Assert.assertEquals(z.dataType(), z2.dataType());
    Assert.assertEquals(x, z2);
    Assert.assertEquals(z, z2);
    Assert.assertEquals(x.data().pointer().address(), z2.data().pointer().address());
    if("CUDA".equalsIgnoreCase(Nd4j.getExecutioner().getEnvironmentInformation().getProperty("backend"))){
        Assert.assertEquals(getDeviceAddress(x), getDeviceAddress(z2));
    }


}
 

public static INDArray scoreArray(INDArray labels, INDArray preOutput) {
    INDArray yhatmag = preOutput.norm2(1);

    INDArray scoreArr = preOutput.mul(labels);
    scoreArr.diviColumnVector(yhatmag);

    return scoreArr;
}
 

/**
 * Calculate the update based on the given gradient
 *
 * @param gradient  the gradient to get the update for
 * @param iteration
 * @return the gradient
 */
@Override
public void applyUpdater(INDArray gradient, int iteration, int epoch) {
    if (m == null || v == null)
        throw new IllegalStateException("Updater has not been initialized with view state");

    double beta1 = config.getBeta1();
    double beta2 = config.getBeta2();
    double learningRate = config.getLearningRate(iteration, epoch);
    double epsilon = config.getEpsilon();

    INDArray oneMinusBeta1Grad = gradient.mul(1.0 - beta1);
    m.muli(beta1).addi(oneMinusBeta1Grad);

    INDArray oneMinusBeta2GradSquared = gradient.mul(gradient).muli(1.0 - beta2);
    v.muli(beta2).addi(oneMinusBeta2GradSquared);

    double beta1t = FastMath.pow(beta1, iteration + 1);

    INDArray biasCorrectedEstimateOfMomentum = m.mul(beta1).divi(1.0 - beta1t);
    INDArray secondTerm = oneMinusBeta1Grad.divi(1 - beta1t);

    INDArray alphat = biasCorrectedEstimateOfMomentum.add(secondTerm).muli(learningRate);

    INDArray sqrtV = Transforms.sqrt(v.dup(gradientReshapeOrder), false).addi(epsilon);

    gradient.assign(alphat).divi(sqrtV);
}
 

@Override
public void applyUpdater(INDArray gradient, int iteration, int epoch) {
    if (m == null || v == null || vHat == null)
        throw new IllegalStateException("Updater has not been initialized with view state");

    double beta1 = config.getBeta1();
    double beta2 = config.getBeta2();
    double learningRate = config.getLearningRate(iteration, epoch);
    double epsilon = config.getEpsilon();

    //m_t = b_1 * m_{t-1} + (1-b_1) * g_t       eq 1 pg 3
    INDArray oneMinusBeta1Grad = gradient.mul(1.0 - beta1);
    m.muli(beta1).addi(oneMinusBeta1Grad);

    //v_t = b_2 * v_{t-1} + (1-b_2) * (g_t)^2   eq 1 pg 3
    INDArray oneMinusBeta2GradSquared = gradient.mul(gradient).muli(1 - beta2);
    v.muli(beta2).addi(oneMinusBeta2GradSquared);

    double beta1t = FastMath.pow(beta1, iteration + 1);
    double beta2t = FastMath.pow(beta2, iteration + 1);

    //vHat_t = max(vHat_{t-1}, v_t)
    Transforms.max(vHat, v, false);

    double alphat = learningRate * FastMath.sqrt(1 - beta2t) / (1 - beta1t);
    if (Double.isNaN(alphat) || alphat == 0.0)
        alphat = epsilon;

    //gradient array contains: sqrt(vHat) + eps
    Nd4j.getExecutioner().execAndReturn(new Sqrt(vHat, gradient)).addi(epsilon);

    //gradient = alphat * m_t / (sqrt(vHat) + eps)
    gradient.rdivi(m).muli(alphat);
}
 

@Override
public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) {
    /*
    //libnd4j only returns diagonal elements, fix in libnd4j?
    //derivative of softmax(in) shape = minibatchxclasses should give minibatch x classes x classes
    int miniBatchSize = in.shape()[0];
    int classSize = in.shape()[1];
    //if (in.rank() != 2) throw exception?
    INDArray z = Nd4j.zeros(miniBatchSize,classSize,classSize);
    INDArray i = Nd4j.eye(classSize);
    INDArray out = z.dup();
    
    //identity matrix extended to 3d
    Nd4j.getExecutioner().execAndReturn(new BroadcastAddOp(z,i,out,new int[] {1,2}));
    
    //D_jS_j = S_i * (delta_ij - S_j)
    Nd4j.getExecutioner().execAndReturn(new BroadcastSubOp(out,in,z,new int[] {0,1}));//1-p or -p
    Nd4j.getExecutioner().execAndReturn(new BroadcastMulOp(z,in,out,new int[] {0,1}));//p*(1-p) or -pi*pj
    
    gradient = out;
    */
    //use loss fn utils and push this for next release
    //        Nd4j.getExecutioner().execAndReturn(new SoftMax(in).derivative());
    //        return in;

    INDArray out = Nd4j.getExecutioner().execAndReturn(new OldSoftMax(in));

    INDArray x = out.mul(epsilon).sum(1);
    INDArray dLdz = out.mul(epsilon.subColumnVector(x));

    return new Pair<>(dLdz, null);
}
 

@Test
public void testEvalAddSelf() {
    /**
     * Note this test fails yet due to needing
     * to validate simple cases like x * x
     * matching number of inputs.
     */
    SameDiff sameDiff = SameDiff.create();
    INDArray arr = Nd4j.linspace(1, 4, 4);
    SDVariable x = sameDiff.var("x", arr);
    SDVariable s = x.mul("s", x);
    INDArray assertion = arr.mul(arr);
    INDArray eval = sameDiff.output(Collections.singletonMap("x", arr), Collections.singletonList("s")).get("s");
    assertEquals(assertion, eval);
}
 

public static void applyNadamUpdater(INDArray gradient, INDArray m, INDArray v, double learningRate, double beta1, double beta2,
                                    double epsilon, int iteration){

    INDArray oneMinusBeta1Grad = gradient.mul(1.0 - beta1);
    m.muli(beta1).addi(oneMinusBeta1Grad);

    INDArray oneMinusBeta2GradSquared = gradient.mul(gradient).muli(1.0 - beta2);
    v.muli(beta2).addi(oneMinusBeta2GradSquared);

    double beta1t = FastMath.pow(beta1, iteration + 1);

    INDArray biasCorrectedEstimateOfMomentum = m.mul(beta1).divi(1.0 - beta1t);
    INDArray secondTerm = oneMinusBeta1Grad.divi(1 - beta1t);

    INDArray alphat = biasCorrectedEstimateOfMomentum.add(secondTerm).muli(learningRate);

    INDArray sqrtV = Transforms.sqrt(v.dup('c'), false).addi(epsilon);

    gradient.assign(alphat).divi(sqrtV);
}
 

@Test
public void testBruteForce() {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
    //X_scaled = X_std * (max - min) + min
    // Dataset features are scaled consecutive natural numbers
    int nSamples = 500;
    int x = 4, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples).reshape(nSamples, 1);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    featureX.muli(x);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    //expected min and max
    INDArray theoreticalMin = Nd4j.create(new double[] {x, y, z}, new long[]{1,3});
    INDArray theoreticalMax = Nd4j.create(new double[] {nSamples * x, nSamples * y, nSamples * z}, new long[]{1,3});
    INDArray theoreticalRange = theoreticalMax.sub(theoreticalMin);

    NormalizerMinMaxScaler myNormalizer = new NormalizerMinMaxScaler();
    myNormalizer.fit(sampleDataSet);

    INDArray minDataSet = myNormalizer.getMin();
    INDArray maxDataSet = myNormalizer.getMax();
    INDArray minDiff = minDataSet.sub(theoreticalMin).max();
    INDArray maxDiff = maxDataSet.sub(theoreticalMax).max();
    assertEquals(minDiff.getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDiff.max().getDouble(0), 0.0, 0.000000001);

    // SAME TEST WITH THE ITERATOR
    int bSize = 1;
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);
    minDataSet = myNormalizer.getMin();
    maxDataSet = myNormalizer.getMax();
    assertEquals(minDataSet.sub(theoreticalMin).max(1).getDouble(0), 0.0, 0.000000001);
    assertEquals(maxDataSet.sub(theoreticalMax).max(1).getDouble(0), 0.0, 0.000000001);

    sampleIter.setPreProcessor(myNormalizer);
    INDArray actual, expected, delta;
    int i = 1;
    while (sampleIter.hasNext()) {
        expected = theoreticalMin.mul(i - 1).div(theoreticalRange);
        actual = sampleIter.next().getFeatures();
        delta = Transforms.abs(actual.sub(expected));
        assertTrue(delta.max(1).getDouble(0) < 0.0001);
        i++;
    }

}
 

/**
 * Non Maximum Suppression - greedily selects the boxes with high confidence. Keep the boxes that have overlap area
 * below the threshold and discards the others.
 *
 * original code:
 *  - https://github.com/kpzhang93/MTCNN_face_detection_alignment/blob/master/code/codes/MTCNNv2/nms.m
 *  - https://github.com/davidsandberg/facenet/blob/master/src/align/detect_face.py#L687
 *
 * @param boxes nd array with bounding boxes: [[x1, y1, x2, y2 score]]
 * @param threshold NMS threshold -  retain overlap <= thresh
 * @param nmsType NMS method to apply. Available values ('Min', 'Union')
 * @return Returns the NMS result
 */
public static INDArray nonMaxSuppression(INDArray boxes, double threshold, NonMaxSuppressionType nmsType) {

	if (boxes.isEmpty()) {
		return Nd4j.empty();
	}

	// TODO Try to prevent following duplications!
	INDArray x1 = boxes.get(all(), point(0)).dup();
	INDArray y1 = boxes.get(all(), point(1)).dup();
	INDArray x2 = boxes.get(all(), point(2)).dup();
	INDArray y2 = boxes.get(all(), point(3)).dup();
	INDArray s = boxes.get(all(), point(4)).dup();

	//area = (x2 - x1 + 1) * (y2 - y1 + 1)
	INDArray area = (x2.sub(x1).add(1)).mul(y2.sub(y1).add(1));

	// sorted_s = np.argsort(s)
	INDArray sortedS = Nd4j.sortWithIndices(s, 0, SORT_ASCENDING)[0];

	INDArray pick = Nd4j.zerosLike(s);
	int counter = 0;

	while (sortedS.size(0) > 0) {

		if (sortedS.size(0) == 1) {
			pick.put(counter++, sortedS.dup());
			break;
		}

		long lastIndex = sortedS.size(0) - 1;
		INDArray i = sortedS.get(point(lastIndex), all()); // last element
		INDArray idx = sortedS.get(interval(0, lastIndex), all()).transpose(); // all until last excluding
		pick.put(counter++, i.dup());

		INDArray xx1 = Transforms.max(x1.get(idx), x1.get(i).getInt(0));
		INDArray yy1 = Transforms.max(y1.get(idx), y1.get(i).getInt(0));
		INDArray xx2 = Transforms.min(x2.get(idx), x2.get(i).getInt(0));
		INDArray yy2 = Transforms.min(y2.get(idx), y2.get(i).getInt(0));

		// w = np.maximum(0.0, xx2 - xx1 + 1)
		// h = np.maximum(0.0, yy2 - yy1 + 1)
		// inter = w * h
		INDArray w = Transforms.max(xx2.sub(xx1).add(1), 0.0f);
		INDArray h = Transforms.max(yy2.sub(yy1).add(1), 0.0f);
		INDArray inter = w.mul(h);

		// if method is 'Min':
		//   o = inter / np.minimum(area[i], area[idx])
		// else:
		//   o = inter / (area[i] + area[idx] - inter)
		int areaI = area.get(i).getInt(0);
		INDArray o = (nmsType == NonMaxSuppressionType.Min) ?
				inter.div(Transforms.min(area.get(idx), areaI)) :
				inter.div(area.get(idx).add(areaI).sub(inter));

		INDArray oIdx = MtcnnUtil.getIndexWhereVector(o, value -> value <= threshold);
		//INDArray oIdx = getIndexWhereVector2(o, Conditions.lessThanOrEqual(threshold));

		if (oIdx.isEmpty()) {
			break;
		}

		sortedS = Nd4j.expandDims(sortedS.get(oIdx), 0).transpose();
	}

	//pick = pick[0:counter]
	return (counter == 0) ? Nd4j.empty() : pick.get(interval(0, counter));
}
 

@Test
public void testBruteForce() {
    //X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
    //X_scaled = X_std * (max - min) + min
    // Dataset features are scaled consecutive natural numbers
    int nSamples = 500;
    int x = 4, y = 2, z = 3;

    INDArray featureX = Nd4j.linspace(1, nSamples, nSamples).reshape(nSamples, 1);
    INDArray featureY = featureX.mul(y);
    INDArray featureZ = featureX.mul(z);
    featureX.muli(x);
    INDArray featureSet = Nd4j.concat(1, featureX, featureY, featureZ);
    INDArray labelSet = Nd4j.zeros(nSamples, 1);
    DataSet sampleDataSet = new DataSet(featureSet, labelSet);

    //expected min and max
    INDArray theoreticalMin = Nd4j.create(new double[] {x, y, z});
    INDArray theoreticalMax = Nd4j.create(new double[] {nSamples * x, nSamples * y, nSamples * z});
    INDArray theoreticalRange = theoreticalMax.sub(theoreticalMin);

    NormalizerMinMaxScaler myNormalizer = new NormalizerMinMaxScaler();
    myNormalizer.fit(sampleDataSet);

    INDArray minDataSet = myNormalizer.getMin();
    INDArray maxDataSet = myNormalizer.getMax();
    INDArray minDiff = minDataSet.sub(theoreticalMin).max(1);
    INDArray maxDiff = maxDataSet.sub(theoreticalMax).max(1);
    assertEquals(minDiff.getDouble(0, 0), 0.0, 0.000000001);
    assertEquals(maxDiff.max(1).getDouble(0, 0), 0.0, 0.000000001);

    // SAME TEST WITH THE ITERATOR
    int bSize = 1;
    DataSetIterator sampleIter = new TestDataSetIterator(sampleDataSet, bSize);
    myNormalizer.fit(sampleIter);
    minDataSet = myNormalizer.getMin();
    maxDataSet = myNormalizer.getMax();
    assertEquals(minDataSet.sub(theoreticalMin).max(1).getDouble(0, 0), 0.0, 0.000000001);
    assertEquals(maxDataSet.sub(theoreticalMax).max(1).getDouble(0, 0), 0.0, 0.000000001);

    sampleIter.setPreProcessor(myNormalizer);
    INDArray actual, expected, delta;
    int i = 1;
    while (sampleIter.hasNext()) {
        expected = theoreticalMin.mul(i - 1).div(theoreticalRange);
        actual = sampleIter.next().getFeatures();
        delta = Transforms.abs(actual.sub(expected));
        assertTrue(delta.max(1).getDouble(0, 0) < 0.0001);
        i++;
    }

}
 

@Test
public void testEvaluationBinaryCustomThreshold() {

    //Sanity check: same results for 0.5 threshold vs. default (no threshold)
    int nExamples = 20;
    int nOut = 2;
    INDArray probs = Nd4j.rand(nExamples, nOut);
    INDArray labels = Nd4j.getExecutioner()
                    .exec(new BernoulliDistribution(Nd4j.createUninitialized(nExamples, nOut), 0.5));

    EvaluationBinary eStd = new EvaluationBinary();
    eStd.eval(labels, probs);

    EvaluationBinary eb05 = new EvaluationBinary(Nd4j.create(new double[] {0.5, 0.5}, new long[]{1,2}));
    eb05.eval(labels, probs);

    EvaluationBinary eb05v2 = new EvaluationBinary(Nd4j.create(new double[] {0.5, 0.5}, new long[]{1,2}));
    for (int i = 0; i < nExamples; i++) {
        eb05v2.eval(labels.getRow(i, true), probs.getRow(i, true));
    }

    for (EvaluationBinary eb2 : new EvaluationBinary[] {eb05, eb05v2}) {
        assertArrayEquals(eStd.getCountTruePositive(), eb2.getCountTruePositive());
        assertArrayEquals(eStd.getCountFalsePositive(), eb2.getCountFalsePositive());
        assertArrayEquals(eStd.getCountTrueNegative(), eb2.getCountTrueNegative());
        assertArrayEquals(eStd.getCountFalseNegative(), eb2.getCountFalseNegative());

        for (int j = 0; j < nOut; j++) {
            assertEquals(eStd.accuracy(j), eb2.accuracy(j), 1e-6);
            assertEquals(eStd.f1(j), eb2.f1(j), 1e-6);
        }
    }


    //Check with decision threshold of 0.25 and 0.125 (for different outputs)
    //In this test, we'll cheat a bit: multiply probabilities by 2 (max of 1.0) and threshold of 0.25 should give
    // an identical result to a threshold of 0.5
    //Ditto for 4x and 0.125 threshold

    INDArray probs2 = probs.mul(2);
    probs2 = Transforms.min(probs2, 1.0);

    INDArray probs4 = probs.mul(4);
    probs4 = Transforms.min(probs4, 1.0);

    EvaluationBinary ebThreshold = new EvaluationBinary(Nd4j.create(new double[] {0.25, 0.125}));
    ebThreshold.eval(labels, probs);

    EvaluationBinary ebStd2 = new EvaluationBinary();
    ebStd2.eval(labels, probs2);

    EvaluationBinary ebStd4 = new EvaluationBinary();
    ebStd4.eval(labels, probs4);

    assertEquals(ebThreshold.truePositives(0), ebStd2.truePositives(0));
    assertEquals(ebThreshold.trueNegatives(0), ebStd2.trueNegatives(0));
    assertEquals(ebThreshold.falsePositives(0), ebStd2.falsePositives(0));
    assertEquals(ebThreshold.falseNegatives(0), ebStd2.falseNegatives(0));

    assertEquals(ebThreshold.truePositives(1), ebStd4.truePositives(1));
    assertEquals(ebThreshold.trueNegatives(1), ebStd4.trueNegatives(1));
    assertEquals(ebThreshold.falsePositives(1), ebStd4.falsePositives(1));
    assertEquals(ebThreshold.falseNegatives(1), ebStd4.falseNegatives(1));
}
 

private static void testIrisMiniBatchGradients(int miniBatchSize, int[] hiddenLayerSizes,
                Activation activationFunction) {
    int totalExamples = 10 * miniBatchSize;
    if (totalExamples > 150) {
        totalExamples = miniBatchSize * (150 / miniBatchSize);
    }
    if (miniBatchSize > 150) {
        fail();
    }
    DataSetIterator iris = new IrisDataSetIterator(miniBatchSize, totalExamples);

    MultiLayerNetwork network = new MultiLayerNetwork(getIrisMLPSimpleConfig(hiddenLayerSizes, Activation.SIGMOID));
    network.init();

    Layer[] layers = network.getLayers();
    int nLayers = layers.length;

    while (iris.hasNext()) {
        DataSet data = iris.next();
        INDArray x = data.getFeatures();
        INDArray y = data.getLabels();

        //Do forward pass:
        INDArray[] layerWeights = new INDArray[nLayers];
        INDArray[] layerBiases = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            layerWeights[i] = layers[i].getParam(DefaultParamInitializer.WEIGHT_KEY).dup();
            layerBiases[i] = layers[i].getParam(DefaultParamInitializer.BIAS_KEY).dup();
        }

        INDArray[] layerZs = new INDArray[nLayers];
        INDArray[] layerActivations = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray layerInput = (i == 0 ? x : layerActivations[i - 1]);
            layerZs[i] = layerInput.castTo(layerWeights[i].dataType()).mmul(layerWeights[i]).addiRowVector(layerBiases[i]);
            layerActivations[i] = (i == nLayers - 1 ? doSoftmax(layerZs[i].dup()) : doSigmoid(layerZs[i].dup()));
        }

        //Do backward pass:
        INDArray[] deltas = new INDArray[nLayers];
        deltas[nLayers - 1] = layerActivations[nLayers - 1].sub(y.castTo(layerActivations[nLayers-1].dataType())); //Out - labels; shape=[miniBatchSize,nOut];
        assertArrayEquals(deltas[nLayers - 1].shape(), new long[] {miniBatchSize, 3});
        for (int i = nLayers - 2; i >= 0; i--) {
            INDArray sigmaPrimeOfZ;
            sigmaPrimeOfZ = doSigmoidDerivative(layerZs[i]);
            INDArray epsilon = layerWeights[i + 1].mmul(deltas[i + 1].transpose()).transpose();
            deltas[i] = epsilon.mul(sigmaPrimeOfZ);
            assertArrayEquals(deltas[i].shape(), new long[] {miniBatchSize, hiddenLayerSizes[i]});
        }

        INDArray[] dLdw = new INDArray[nLayers];
        INDArray[] dLdb = new INDArray[nLayers];
        for (int i = 0; i < nLayers; i++) {
            INDArray prevActivations = (i == 0 ? x : layerActivations[i - 1]);
            //Raw gradients, so not yet divided by mini-batch size (division is done in BaseUpdater)
            dLdw[i] = deltas[i].transpose().castTo(prevActivations.dataType()).mmul(prevActivations).transpose(); //Shape: [nIn, nOut]
            dLdb[i] = deltas[i].sum(true, 0); //Shape: [1,nOut]

            int nIn = (i == 0 ? 4 : hiddenLayerSizes[i - 1]);
            int nOut = (i < nLayers - 1 ? hiddenLayerSizes[i] : 3);
            assertArrayEquals(dLdw[i].shape(), new long[] {nIn, nOut});
            assertArrayEquals(dLdb[i].shape(), new long[] {1, nOut});
        }


        //Calculate and get gradient, compare to expected
        network.setInput(x);
        network.setLabels(y);
        network.computeGradientAndScore();
        Gradient gradient = network.gradientAndScore().getFirst();

        float eps = 1e-4f;
        for (int i = 0; i < hiddenLayerSizes.length; i++) {
            String wKey = i + "_" + DefaultParamInitializer.WEIGHT_KEY;
            String bKey = i + "_" + DefaultParamInitializer.BIAS_KEY;
            INDArray wGrad = gradient.getGradientFor(wKey);
            INDArray bGrad = gradient.getGradientFor(bKey);
            float[] wGradf = asFloat(wGrad);
            float[] bGradf = asFloat(bGrad);
            float[] expWGradf = asFloat(dLdw[i]);
            float[] expBGradf = asFloat(dLdb[i]);
            assertArrayEquals(wGradf, expWGradf, eps);
            assertArrayEquals(bGradf, expBGradf, eps);
        }
    }
}