org.nd4j.linalg.indexing.BooleanIndexing Java Examples

@Test
public void testBinomialDistribution1() {
    Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
    Random random2 = Nd4j.getRandomFactory().getNewRandomInstance(119);

    INDArray z1 = Nd4j.zeros(1000);
    INDArray z2 = Nd4j.zeros(1000);
    INDArray z1Dup = Nd4j.zeros(1000);

    BinomialDistribution op1 = new BinomialDistribution(z1, 5, 0.25);
    BinomialDistribution op2 = new BinomialDistribution(z2, 5, 0.25);

    Nd4j.getExecutioner().exec(op1, random1);
    Nd4j.getExecutioner().exec(op2, random2);

    assertNotEquals(z1Dup, z1);

    assertEquals(z1, z2);

    BooleanIndexing.and(z1, Conditions.lessThanOrEqual(5.0));
    BooleanIndexing.and(z1, Conditions.greaterThanOrEqual(0.0));
}
 

@Override
public INDArray gradient(INDArray x, INDArray preOutDistributionParams) {
    INDArray output = preOutDistributionParams.dup();
    activationFn.getActivation(output, true);
    x = x.castTo(preOutDistributionParams.dataType());

    INDArray diff = x.sub(output);
    INDArray outOneMinusOut = output.rsub(1.0).muli(output);

    INDArray grad = diff.divi(outOneMinusOut);
    grad = activationFn.backprop(preOutDistributionParams.dup(), grad).getFirst();

    //Issue: if output == 0 or output == 1, then (assuming sigmoid output or similar)
    //sigmaPrime == 0, sigmaPrime * (x-out) / (out*(1-out)) == 0 * (x-out) / 0 -> 0/0 -> NaN. But taking limit, we want
    //0*(x-out)/0 == 0 -> implies 0 gradient at the far extremes (0 or 1) of the output
    BooleanIndexing.replaceWhere(grad, 0.0, Conditions.isNan());
    return grad.negi();
}
 

@Override
public INDArray getActivation(INDArray in, boolean training) {
    if (training) {
        try(MemoryWorkspace ws = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
            this.alpha = Nd4j.rand(in.shape(), l, u, Nd4j.getRandom());
        }
        INDArray inTimesAlpha = in.mul(alpha);
        BooleanIndexing.replaceWhere(in, inTimesAlpha, Conditions.lessThan(0));
    } else {
        this.alpha = null;
        double a = 0.5 * (l + u);
        return Nd4j.getExecutioner().execAndReturn(new RectifedLinear(in, a));
    }

    return in;
}
 

private INDArray calculateBernoulli(INDArray minorityLabels, INDArray labelMask, double targetMinorityDist) {

        INDArray minorityClass = minorityLabels.dup().muli(labelMask);
        INDArray majorityClass = Transforms.not(minorityLabels).muli(labelMask);

        //all minorityLabel class, keep masks as is
        //presence of minoriy class and donotmask minority windows set to true return label as is
        if (majorityClass.sumNumber().intValue() == 0
                        || (minorityClass.sumNumber().intValue() > 0 && donotMaskMinorityWindows))
            return labelMask;
        //all majority class and set to not mask all majority windows sample majority class by 1-targetMinorityDist
        if (minorityClass.sumNumber().intValue() == 0 && !maskAllMajorityWindows)
            return labelMask.muli(1 - targetMinorityDist);

        //Probabilities to be used for bernoulli sampling
        INDArray minoritymajorityRatio = minorityClass.sum(1).div(majorityClass.sum(1));
        INDArray majorityBernoulliP = minoritymajorityRatio.muli(1 - targetMinorityDist).divi(targetMinorityDist);
        BooleanIndexing.replaceWhere(majorityBernoulliP, 1.0, Conditions.greaterThan(1.0)); //if minority ratio is already met round down to 1.0
        return majorityClass.muliColumnVector(majorityBernoulliP).addi(minorityClass);
    }
 

private INDArray calculateBernoulli(INDArray minorityLabels, INDArray labelMask, double targetMinorityDist) {

        INDArray minorityClass = minorityLabels.castTo(Nd4j.defaultFloatingPointType()).muli(labelMask);
        INDArray majorityClass = minorityLabels.rsub(1.0).muli(labelMask);      //rsub(1.0) is equivalent to swapping 0s and 1s

        //all minorityLabel class, keep masks as is
        //presence of minoriy class and donotmask minority windows set to true return label as is
        if (majorityClass.sumNumber().intValue() == 0
                        || (minorityClass.sumNumber().intValue() > 0 && donotMaskMinorityWindows))
            return labelMask;
        //all majority class and set to not mask all majority windows sample majority class by 1-targetMinorityDist
        if (minorityClass.sumNumber().intValue() == 0 && !maskAllMajorityWindows)
            return labelMask.muli(1 - targetMinorityDist);

        //Probabilities to be used for bernoulli sampling
        INDArray minoritymajorityRatio = minorityClass.sum(1).div(majorityClass.sum(1));
        INDArray majorityBernoulliP = minoritymajorityRatio.muli(1 - targetMinorityDist).divi(targetMinorityDist);
        BooleanIndexing.replaceWhere(majorityBernoulliP, 1.0, Conditions.greaterThan(1.0)); //if minority ratio is already met round down to 1.0
        return majorityClass.muliColumnVector(majorityBernoulliP).addi(minorityClass);
    }
 

@Test
public void testBinomialDistribution2() {
    Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
    Random random2 = Nd4j.getRandomFactory().getNewRandomInstance(119);

    INDArray z1 = Nd4j.create(DataType.FLOAT, 1000);
    INDArray z2 = Nd4j.zeros(DataType.FLOAT,1000);
    INDArray z1Dup = Nd4j.zeros(DataType.FLOAT,1000);

    INDArray probs = Nd4j.create(new float[] {0.25f, 0.43f, 0.55f, 0.43f, 0.25f});

    BinomialDistribution op1 = new BinomialDistribution(z1, 5, probs);
    BinomialDistribution op2 = new BinomialDistribution(z2, 5, probs);

    Nd4j.getExecutioner().exec(op1, random1);
    Nd4j.getExecutioner().exec(op2, random2);

    assertNotEquals(z1Dup, z1);

    assertEquals(z1, z2);

    BooleanIndexing.and(z1, Conditions.lessThanOrEqual(5.0));
    BooleanIndexing.and(z1, Conditions.greaterThanOrEqual(0.0));
}
 

private static INDArray relError(@NonNull INDArray a1, @NonNull INDArray a2, double minAbsError){
    long numNaN1 = Nd4j.getExecutioner().exec(new MatchCondition(a1, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
    long numNaN2 = Nd4j.getExecutioner().exec(new MatchCondition(a2, Conditions.isNan(), Integer.MAX_VALUE)).getInt(0);
    Preconditions.checkState(numNaN1 == 0, "Array 1 has NaNs");
    Preconditions.checkState(numNaN2 == 0, "Array 2 has NaNs");

    INDArray abs1 = Transforms.abs(a1, true);
    INDArray abs2 = Transforms.abs(a2, true);
    INDArray absDiff = Transforms.abs(a1.sub(a2), false);

    //abs(a1-a2) < minAbsError ? 1 : 0
    INDArray greaterThanMinAbs = Transforms.abs(a1.sub(a2), false);
    BooleanIndexing.replaceWhere(greaterThanMinAbs, 0.0, Conditions.lessThan(minAbsError));
    BooleanIndexing.replaceWhere(greaterThanMinAbs, 1.0, Conditions.greaterThan(0.0));

    INDArray result = absDiff.divi(abs1.add(abs2));
    //Only way to have NaNs given there weren't any in original : both 0s
    BooleanIndexing.replaceWhere(result, 0.0, Conditions.isNan());
    //Finally, set to 0 if less than min abs error, or unchanged otherwise
    result.muli(greaterThanMinAbs);

    return result;
}
 

@Override
public INDArray getActivation(INDArray in, boolean training) {
    if (training) {
        try(MemoryWorkspace ignored = Nd4j.getWorkspaceManager().scopeOutOfWorkspaces()) {
            this.alpha = Nd4j.rand(l, u, Nd4j.getRandom(), in.shape());
        }
        INDArray inTimesAlpha = in.mul(alpha);
        BooleanIndexing.replaceWhere(in, inTimesAlpha, Conditions.lessThan(0));
    } else {
        this.alpha = null;
        double a = 0.5 * (l + u);
        return Nd4j.getExecutioner().exec(new RectifiedLinear(in, a));
    }

    return in;
}
 

@Test
public void testReplaceWhereScalar() {
    for (Condition c : new Condition[]{Conditions.lessThan(0.5), Conditions.greaterThan(0.5), Conditions.equals(0.5)}) {

        log.info("Testing condition: " + c.getClass().getSimpleName());
        INDArray inArr = Nd4j.rand(DataType.DOUBLE, 3, 4);
        SameDiff sd = SameDiff.create();
        SDVariable in = sd.var("in", inArr);
        SDVariable where = sd.replaceWhere(in, 10, c);

        INDArray exp = inArr.dup();
        BooleanIndexing.replaceWhere(exp, 10, c);

        SDVariable loss = where.std(true);

        TestCase tc = new TestCase(sd);

        String err = OpValidation.validate(tc);
        assertNull(err);
    }
}
 

@Test
public void testReplaceWhereArray() {
    for (Condition c : new Condition[]{Conditions.lessThan(0.5), Conditions.greaterThan(0.5), Conditions.equals(0.5)}) {

        INDArray inArr = Nd4j.rand(3, 4);
        INDArray inArr2 = Nd4j.valueArrayOf(3, 4, 10);
        SameDiff sd = SameDiff.create();
        SDVariable in = sd.var("in", inArr);
        SDVariable in2 = sd.var("in2", inArr2);
        SDVariable where = sd.replaceWhere(in, in2, c);

        INDArray exp = inArr.dup();
        BooleanIndexing.replaceWhere(exp, inArr2, c);

        SDVariable loss = where.std(true);

        TestCase tc = new TestCase(sd);

        String err = OpValidation.validate(tc);
        assertNull(err);
    }
}
 

@Test
public void testBinomialDistribution2() throws Exception {
    Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
    Random random2 = Nd4j.getRandomFactory().getNewRandomInstance(119);

    INDArray z1 = Nd4j.zeros(1000);
    INDArray z2 = Nd4j.zeros(1000);
    INDArray z1Dup = Nd4j.zeros(1000);

    INDArray probs = Nd4j.create(new float[] {0.25f, 0.43f, 0.55f, 0.43f, 0.25f});

    BinomialDistribution op1 = new BinomialDistribution(z1, 5, probs);
    BinomialDistribution op2 = new BinomialDistribution(z2, 5, probs);

    Nd4j.getExecutioner().exec(op1, random1);
    Nd4j.getExecutioner().exec(op2, random2);

    assertNotEquals(z1Dup, z1);

    assertEquals(z1, z2);

    BooleanIndexing.and(z1, Conditions.lessThanOrEqual(5.0));
    BooleanIndexing.and(z1, Conditions.greaterThanOrEqual(0.0));
}
 

@Test
public void testBinomialDistribution1() throws Exception {
    Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
    Random random2 = Nd4j.getRandomFactory().getNewRandomInstance(119);

    INDArray z1 = Nd4j.zeros(1000);
    INDArray z2 = Nd4j.zeros(1000);
    INDArray z1Dup = Nd4j.zeros(1000);

    BinomialDistribution op1 = new BinomialDistribution(z1, 5, 0.25);
    BinomialDistribution op2 = new BinomialDistribution(z2, 5, 0.25);

    Nd4j.getExecutioner().exec(op1, random1);
    Nd4j.getExecutioner().exec(op2, random2);

    assertNotEquals(z1Dup, z1);

    assertEquals(z1, z2);

    BooleanIndexing.and(z1, Conditions.lessThanOrEqual(5.0));
    BooleanIndexing.and(z1, Conditions.greaterThanOrEqual(0.0));
}
 

@Override
public INDArray bucket(INDArray query) {
    INDArray queryRes = rawBucketOf(query);

    if(numTables > 1) {
        INDArray entropyQueries = entropy(query);

        // loop, addi + conditionalreplace -> poor man's OR function
        for (int i = 0; i < numTables; i++) {
            INDArray row = entropyQueries.getRow(i, true);
            queryRes.addi(rawBucketOf(row));
        }
        BooleanIndexing.replaceWhere(queryRes, 1.0, Conditions.greaterThan(0.0));
    }

    return queryRes;
}
 

/**
 * This is special test that checks for memory alignment
 * @throws Exception
 */
@Test
@Ignore
public void testDTypeSpam() throws Exception {
    Random rnd = new Random();
    for(int i = 0; i < 100; i++) {
        DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
        float rand[] = new float[rnd.nextInt(10) + 1];
        for (int x = 0; x < rand.length; x++) {
            rand[x] = rnd.nextFloat();
        }
        Nd4j.getConstantHandler().getConstantBuffer(rand);

        int shape[] = new int[rnd.nextInt(3)+2];
        for (int x = 0; x < shape.length; x++) {
            shape[x] = rnd.nextInt(100) + 2;
        }

        DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
        INDArray array = Nd4j.rand(shape);
        BooleanIndexing.applyWhere(array, Conditions.lessThan(rnd.nextDouble()), rnd.nextDouble());
    }
}
 

private static INDArray filteredStd(DistributionStats stats) {
    /*
        To avoid division by zero when the std deviation is zero, replace zeros by one
     */
    INDArray stdCopy = stats.getStd();
    BooleanIndexing.replaceWhere(stdCopy, 1.0, Conditions.equals(0));
    return stdCopy;
}
 

@Override
public INDArray getActivation(INDArray in, boolean training) {
    // no support in ELU native to override alpha
    if (this.alpha != 1.00) {
        INDArray alphaMultiple = Nd4j.getExecutioner().execAndReturn(new ELU(in.dup()));
        alphaMultiple.muli(alpha);
        BooleanIndexing.replaceWhere(in, alphaMultiple, Conditions.lessThan(0));
    } else {
        Nd4j.getExecutioner().execAndReturn(new ELU(in));
    }
    return in;
}
 

@Override
public boolean remove(Object o) {
    int idx = BooleanIndexing.firstIndex(container,new EqualsCondition((double) o)).getInt(0);
    if(idx < 0)
        return false;
    container.put(new INDArrayIndex[]{NDArrayIndex.interval(idx,container.length())},container.get(NDArrayIndex.interval(idx + 1,container.length())));
    container = container.reshape(1,size);
    return true;
}
 

@Override
public void processResidual(int iteration, int epoch, double lastThreshold, INDArray residualVector) {
    if(iteration > 0 && iteration % frequency == 0) {
        double currClip = lastThreshold * thresholdMultipleClipValue;
        //TODO replace with single op once we have GPU version
        BooleanIndexing.replaceWhere(residualVector, currClip, Conditions.greaterThan(currClip));
        BooleanIndexing.replaceWhere(residualVector, -currClip, Conditions.lessThan(-currClip));
        log.debug("Applied residual clipping: iter={}, epoch={}, lastThreshold={}, multiple={}, clipValue={}", iteration, epoch, lastThreshold, thresholdMultipleClipValue, currClip);
    }
}
 

/**
 * Extract out the last time steps (2d array from 3d array input) accounting for the mask layer, if present.
 *
 * @param pullFrom Input time series array (rank 3) to pull the last time steps from
 * @param mask     Mask array (rank 2). May be null
 * @return         2d array of the last time steps
 */
public static Pair<INDArray,int[]> pullLastTimeSteps(INDArray pullFrom, INDArray mask){
    //Then: work out, from the mask array, which time step of activations we want, extract activations
    //Also: record where they came from (so we can do errors later)
    int[] fwdPassTimeSteps;
    INDArray out;
    if (mask == null) {

        //No mask array -> extract same (last) column for all
        long lastTS = pullFrom.size(2) - 1;
        out = pullFrom.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(lastTS));
        fwdPassTimeSteps = null; //Null -> last time step for all examples
    } else {
        val outShape = new long[] {pullFrom.size(0), pullFrom.size(1)};
        out = Nd4j.create(outShape);

        //Want the index of the last non-zero entry in the mask array
        INDArray lastStepArr = BooleanIndexing.lastIndex(mask, Conditions.epsNotEquals(0.0), 1);
        fwdPassTimeSteps = lastStepArr.data().asInt();

        //Now, get and assign the corresponding subsets of 3d activations:
        for (int i = 0; i < fwdPassTimeSteps.length; i++) {
            //TODO can optimize using reshape + pullRows
            out.putRow(i, pullFrom.get(NDArrayIndex.point(i), NDArrayIndex.all(),
                    NDArrayIndex.point(fwdPassTimeSteps[i])));
        }
    }

    return new Pair<>(out, fwdPassTimeSteps);
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if (labels.size(1) != preOutput.size(1)) {
        throw new IllegalArgumentException(
                        "Labels array numColumns (size(1) = " + labels.size(1) + ") does not match output layer"
                                        + " number of outputs (nOut = " + preOutput.size(1) + ") ");

    }
    /*
    gradient is 0 if yhaty is >= 1
    else gradient is gradient of the loss function = (1-yhaty) wrt preOutput = -y*derivative_of_yhat wrt preout
    */

    INDArray bitMaskRowCol = scoreArray(labels, preOutput, activationFn, mask);
    /*
        bit mask is 0 if 1-sigma(y*yhat) is neg
        bit mask is 1 if 1-sigma(y*yhat) is +ve
     */
    BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0));
    BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));

    INDArray dLda = labels.neg().muli(bitMaskRowCol);

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with parameters

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public boolean remove(Object o) {
    int idx = BooleanIndexing.firstIndex(container,new EqualsCondition((double) o)).getInt(0);
    if(idx < 0)
        return false;
    container.put(new INDArrayIndex[]{NDArrayIndex.interval(idx,container.length())},container.get(NDArrayIndex.interval(idx + 1,container.length())));
    container = container.reshape(size);
    return true;
}
 

private INDArray calcLogProbArray(INDArray x, INDArray preOutDistributionParams) {
    x = x.castTo(preOutDistributionParams.dataType());
    INDArray output = preOutDistributionParams.dup();
    activationFn.getActivation(output, false);

    INDArray logOutput = Transforms.log(output, true);
    INDArray log1SubOut = Transforms.log(output.rsubi(1.0), false);

    //For numerical stability: if output = 0, then log(output) == -infinity
    //then x * log(output) = NaN, but lim(x->0, output->0)[ x * log(output) ] == 0
    // therefore: want 0*log(0) = 0, NOT 0*log(0) = NaN by default
    BooleanIndexing.replaceWhere(logOutput, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0
    BooleanIndexing.replaceWhere(log1SubOut, 0.0, Conditions.isInfinite()); //log(out)= +/- inf -> x == 0.0 -> 0 * log(0) = 0
    return logOutput.muli(x).addi(x.rsub(1.0).muli(log1SubOut));
}
 

private static INDArray filteredStd(DistributionStats stats) {
    /*
        To avoid division by zero when the std deviation is zero, replace zeros by one
     */
    INDArray stdCopy = stats.getStd();
    BooleanIndexing.replaceWhere(stdCopy, 1.0, Conditions.equals(0));
    return stdCopy;
}
 

@Override
public void preProcess(DataSet toPreProcess) {

    INDArray label3d = toPreProcess.getLabels();
    Preconditions.checkState(label3d.rank() == 3, "LabelLastTimeStepPreProcessor expects rank 3 labels, got rank %s labels with shape %ndShape", label3d.rank(), label3d);

    INDArray lMask = toPreProcess.getLabelsMaskArray();
    //If no mask: assume that examples for each minibatch are all same length
    INDArray labels2d;
    if(lMask == null){
        labels2d = label3d.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(label3d.size(2)-1)).dup();
    } else {
        //Use the label mask to work out the last time step...
        INDArray lastIndex = BooleanIndexing.lastIndex(lMask, Conditions.greaterThan(0), 1);
        long[] idxs = lastIndex.data().asLong();

        //Now, extract out:
        labels2d = Nd4j.create(DataType.FLOAT, label3d.size(0), label3d.size(1));

        //Now, get and assign the corresponding subsets of 3d activations:
        for (int i = 0; i < idxs.length; i++) {
            long lastStepIdx = idxs[i];
            Preconditions.checkState(lastStepIdx >= 0, "Invalid last time step index: example %s in minibatch is entirely masked out" +
                    " (label mask is all 0s, meaning no label data is present for this example)", i);
            //TODO can optimize using reshape + pullRows
            labels2d.putRow(i, label3d.get(NDArrayIndex.point(i), NDArrayIndex.all(), NDArrayIndex.point(lastStepIdx)));
        }
    }

    toPreProcess.setLabels(labels2d);
    toPreProcess.setLabelsMaskArray(null);  //Remove label mask if present
}
 

@Override
public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) {
    assertShape(in, epsilon);
    INDArray dLdz = Nd4j.ones(in.shape());
    BooleanIndexing.replaceWhere(dLdz, alpha, Conditions.lessThanOrEqual(0.0));
    dLdz.muli(epsilon);

    return new Pair<>(dLdz, null);
}
 

protected INDArray scoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype

    INDArray output = activationFn.getActivation(preOutput.dup(), true);
    if(activationFn instanceof ActivationSoftmax && softmaxClipEps > 0.0){
        BooleanIndexing.replaceWhere(output, softmaxClipEps, Conditions.lessThan(softmaxClipEps));
        BooleanIndexing.replaceWhere(output, 1.0-softmaxClipEps, Conditions.greaterThan(1.0-softmaxClipEps));
    }
    INDArray scoreArr = Transforms.log(output, false).muli(labels);

    //Weighted loss function
    if (weights != null) {
        if (weights.length() != scoreArr.size(1)) {
            throw new IllegalStateException("Weights vector (length " + weights.length()
                            + ") does not match output.size(1)=" + preOutput.size(1));
        }
        scoreArr.muliRowVector(weights.castTo(scoreArr.dataType()));
    }

    if (mask != null) {
        LossUtil.applyMask(scoreArr, mask);
    }
    return scoreArr;
}
 

@Override
public INDArray computeScoreArray(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);
    BooleanIndexing.replaceWhere(scoreArr, 0.0, Conditions.lessThan(0.0));//max(0,1-y*yhat)
    scoreArr.muli(scoreArr);
    return scoreArr.sum(true,1);
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype
    INDArray scoreArr = scoreArray(labels, preOutput, activationFn, mask);

    INDArray bitMaskRowCol = scoreArr.dup();
    /*
        bit mask is 0 if 1-sigma(y*yhat) is neg, bit mask is 1 if 1-sigma(y*yhat) is +ve
     */
    BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0));
    BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));

    INDArray dLda = scoreArr.muli(2).muli(labels.neg());
    dLda.muli(bitMaskRowCol);

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with params

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}
 

@Override
public void apply(INDArray param){
    INDArray norm = param.norm2(dimensions);
    INDArray clipped = norm.unsafeDuplication();
    BooleanIndexing.replaceWhere(clipped, maxNorm, Conditions.greaterThan(maxNorm));
    norm.addi(epsilon);
    clipped.divi(norm);

    Broadcast.mul(param, clipped, param, getBroadcastDims(dimensions, param.rank()) );
}
 

@Override
public INDArray computeGradient(INDArray labels, INDArray preOutput, IActivation activationFn, INDArray mask) {
    if(!labels.equalShapes(preOutput)){
        Preconditions.throwEx("Labels and preOutput must have equal shapes: got shapes %s vs %s", labels.shape(), preOutput.shape());
    }
    labels = labels.castTo(preOutput.dataType());   //No-op if already correct dtype

    /*
    gradient is 0 if yhaty is >= 1
    else gradient is gradient of the loss function = (1-yhaty) wrt preOutput = -y*derivative_of_yhat wrt preout
    */

    INDArray bitMaskRowCol = scoreArray(labels, preOutput, activationFn, mask);
    /*
        bit mask is 0 if 1-sigma(y*yhat) is neg
        bit mask is 1 if 1-sigma(y*yhat) is +ve
     */
    BooleanIndexing.replaceWhere(bitMaskRowCol, 0.0, Conditions.lessThan(0.0));
    BooleanIndexing.replaceWhere(bitMaskRowCol, 1.0, Conditions.greaterThan(0.0));

    INDArray dLda = labels.neg().muli(bitMaskRowCol);

    if (mask != null && LossUtil.isPerOutputMasking(dLda, mask)) {
        //For *most* activation functions: we don't actually need to mask dL/da in addition to masking dL/dz later
        //but: some, like softmax, require both (due to dL/dz_i being a function of dL/da_j, for i != j)
        //We could add a special case for softmax (activationFn instanceof ActivationSoftmax) but that would be
        // error prone - though buy us a tiny bit of performance
        LossUtil.applyMask(dLda, mask);
    }

    INDArray gradients = activationFn.backprop(preOutput, dLda).getFirst(); //TODO activation functions with parameters

    if (mask != null) {
        LossUtil.applyMask(gradients, mask);
    }

    return gradients;
}