org.deeplearning4j.nn.workspace.ArrayType Java Examples

public static INDArray reshapeCnn3dMask(@NonNull Convolution3D.DataFormat format, INDArray mask, INDArray label, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(mask == null)
        return null;
    Preconditions.checkState(mask.rank() == 5, "Expected rank 5 mask for Cnn3DLossLayer in a shape broadcastable to labels shape:" +
            " got mask shape %ndShape with label shape %ndShape", mask, label);

    if(mask.equalShapes(label) ||
            (format == Convolution3D.DataFormat.NDHWC && mask.size(0) == label.size(0) && mask.size(1) == label.size(1) && mask.size(2) == label.size(2) && mask.size(3) == label.size(3)) ||
            (format == Convolution3D.DataFormat.NDHWC && mask.size(0) == label.size(0) && mask.size(2) == label.size(2) && mask.size(3) == label.size(3) && mask.size(4) == label.size(4))) {
        //Already OK shape for reshaping
        return reshape5dTo2d(format, mask, workspaceMgr, type);
    } else {
        //Need to broadcast first
        long[] lShape = label.shape().clone();
        int channelIdx = format == Convolution3D.DataFormat.NCDHW ? 1 : 4;
        lShape[channelIdx] = mask.size(channelIdx);     //Keep existing channel size

        INDArray bMask = workspaceMgr.createUninitialized(type, mask.dataType(), lShape, 'c');
        Nd4j.exec(new Assign(new INDArray[]{bMask, mask}, new INDArray[]{bMask}));
        return reshape5dTo2d(format, bMask, workspaceMgr, type);
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray input, INDArray epsilon, double k, double n, double alpha, double beta, LayerWorkspaceMgr workspaceMgr) {
    INDArray gradAtInput = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), input.shape());

    if(context == null){
        context = Nd4j.getExecutioner().buildContext();
        context.setTArguments(k, alpha, beta);
        context.setIArguments((int)n);
    } else
        context.purge();

    LocalResponseNormalization op = new LocalResponseNormalization();

    context.setInputArray(0, input);
    context.setInputArray(0, epsilon);
    context.setOutputArray(0, gradAtInput);

    Nd4j.exec(op, context);
    Gradient g = new DefaultGradient();
    return new Pair<>(g, gradAtInput);
}
 

@Override
public INDArray activate(INDArray x, boolean training, double k, double n, double alpha, double beta, LayerWorkspaceMgr workspaceMgr) {
    INDArray out = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, x.dataType(), x.shape());

    if(context == null){
        context = Nd4j.getExecutioner().buildContext();
        context.setTArguments(k, alpha, beta);
        context.setIArguments((int)n);
    } else
        context.purge();

    context.setInputArray(0, x);
    context.setOutputArray(0, out);

    LocalResponseNormalization op = new LocalResponseNormalization();

    Nd4j.exec(op, context);
    return out;
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr mgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, mgr);

    INDArray in;
    if (training) {
        in = mgr.dup(ArrayType.ACTIVATIONS, input, input.ordering());
    } else {
        in = mgr.leverageTo(ArrayType.ACTIVATIONS, input);
    }

    INDArray alpha = getParam(PReLUParamInitializer.WEIGHT_KEY);

    return new ActivationPReLU(alpha, axes).getActivation(in, training);
}
 

@Override
public INDArray doForward(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoForward())
        throw new IllegalStateException("Cannot do forward pass: inputs not set (L2NormalizeVertex " + vertexName
                        + " idx " + vertexIndex + ")");

    // L2 norm along all dimensions except 0, unless user-specified
    // x / |x|2
    INDArray x = inputs[0];
    int[] dimensions = getDimensions(x);

    INDArray xNorm2 = x.norm2(dimensions);
    Transforms.max(xNorm2, eps, false);
    try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATIONS)){
        if (x.rank() == 2) {
            return x.divColumnVector(xNorm2);
        } else {
            INDArray out = Nd4j.createUninitialized(x.shape(), x.ordering());
            return Nd4j.getExecutioner().exec(new BroadcastDivOp(x, xNorm2, out, 0));
        }
    }
}
 

@Test
public void testReverse(){

    for(char c : new char[]{'f','c'}) {

        INDArray in = Nd4j.linspace(1, 3 * 5 * 10, 3 * 5 * 10, Nd4j.dataType()).reshape('f', 3, 5, 10).dup(c);
        INDArray inMask = Nd4j.linspace(1, 30, 30, Nd4j.dataType()).reshape('f', 3, 10).dup(c); //Minibatch, TS length

        INDArray inReverseExp = reverseTimeSeries(in);
        INDArray inMaskReverseExp = Nd4j.create(inMask.shape());
        for (int i = 0; i < inMask.size(1); i++) {
            inMaskReverseExp.putColumn(i, inMask.getColumn(inMask.size(1) - i - 1));
        }

        INDArray inReverse = TimeSeriesUtils.reverseTimeSeries(in, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT);
        INDArray inMaskReverse = TimeSeriesUtils.reverseTimeSeriesMask(inMask, LayerWorkspaceMgr.noWorkspaces(), ArrayType.INPUT);

        assertEquals(inReverseExp, inReverse);
        assertEquals(inMaskReverseExp, inMaskReverse);
    }
}
 

protected void validateArrayWorkspaces(LayerWorkspaceMgr mgr, INDArray array, ArrayType arrayType, int layerIdx,
                                       boolean isPreprocessor, String op){
    try{
        mgr.validateArrayLocation(arrayType, array, false, layerIdx > 0);
    } catch (ND4JWorkspaceException e){
        String layerName = layers[layerIdx].conf().getLayer().getLayerName();
        String clazz;
        if(isPreprocessor){
            clazz = layerWiseConfigurations.getInputPreProcess(layerIdx).getClass().getName();
        } else {
            clazz = layers[layerIdx].getClass().getName();
        }
        throw new IllegalStateException(op + ": array (" + arrayType + ") workspace validation failed (" +
                (isPreprocessor ? "preprocessor" : "layer ") + layerIdx + (layerName != null ? " - layer name \"" +
                layerName + "\"" : "") + " - class: " + clazz + ") - array is defined in incorrect workspace", e);
    }
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (output == null)
        return null; //In a few cases: output may be null, and this is valid. Like time series data -> embedding layer
    //Need to reshape FeedForward layer epsilons (2d) to 3d (for use in RNN layer backprop calculations)
    if (output.rank() != 2)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 2 (i.e., epsilons from feed forward layer)");
    if (output.ordering() != 'f' || !Shape.hasDefaultStridesForShape(output))
        output = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, output, 'f');

    val shape = output.shape();
    INDArray reshaped = output.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    if (rnnDataFormat == RNNFormat.NCW){
        reshaped = reshaped.permute(0, 2, 1);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, reshaped);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Need to reshape FF activations (2d) activations to 3d (for input into RNN layer)
    if (input.rank() != 2)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 2 (i.e., activations for FF layer)");
    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'f');

    val shape = input.shape();
    INDArray reshaped = input.reshape('f', miniBatchSize, shape[0] / miniBatchSize, shape[1]);
    if (rnnDataFormat == RNNFormat.NCW){
        reshaped = reshaped.permute(0, 2, 1);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, reshaped);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);

    if(epsilon.dataType() != dataType){
        epsilon = epsilon.castTo(dataType);
    }

    INDArray outEpsilon;
    try(MemoryWorkspace ws = workspaceMgr.notifyScopeBorrowed(ArrayType.ACTIVATION_GRAD)){
        if (layerConf().getDataFormat() == RNNFormat.NCW) {
            outEpsilon = epsilon.sum(2);
        }else{
            outEpsilon = epsilon.sum(1);
        }
    }

    Gradient gradient = new DefaultGradient();
    return new Pair<>(gradient, outEpsilon);
}
 

@Override
public INDArray applyDropout(INDArray inputActivations, INDArray output, int iteration, int epoch, LayerWorkspaceMgr workspaceMgr) {
    Preconditions.checkArgument(inputActivations.rank() == 5 || inputActivations.rank() == 4
            || inputActivations.rank() == 3, "Cannot apply spatial dropout to activations of rank %s: " +
            "spatial dropout can only be used for rank 3, 4 or 5 activations (input activations shape: %s)"
            , inputActivations.rank(), inputActivations.shape());

    double currP;
    if (pSchedule != null) {
        currP = pSchedule.valueAt(iteration, epoch);
    } else {
        currP = p;
    }

    val minibatch = inputActivations.size(0);
    val dim1 = inputActivations.size(1);
    mask = workspaceMgr.createUninitialized(ArrayType.INPUT, output.dataType(), minibatch, dim1).assign(1.0);
    Nd4j.getExecutioner().exec(new DropOutInverted(mask, currP));

    Broadcast.mul(inputActivations, mask, output, 0, 1);
    return output;
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);

    if (cacheMode == null)
        cacheMode = CacheMode.NONE;

    INDArray z = preOutput(training, false, workspaceMgr);
    if (training && cacheMode != CacheMode.NONE && workspaceMgr.hasConfiguration(ArrayType.FF_CACHE)
            && workspaceMgr.isWorkspaceOpen(ArrayType.FF_CACHE)) {
        try (MemoryWorkspace wsB = workspaceMgr.notifyScopeBorrowed(ArrayType.FF_CACHE)) {
            preOutput = z.unsafeDuplication();
        }
    }
    return z;
}
 

@Override
public INDArray getParameter(Layer layer, String paramKey, int iteration, int epoch, boolean train, LayerWorkspaceMgr workspaceMgr) {
    ParamInitializer init = layer.conf().getLayer().initializer();
    INDArray param = layer.getParam(paramKey);

    double p;
    if(weightRetainProbSchedule == null){
        p = weightRetainProb;
    } else {
        p = weightRetainProbSchedule.valueAt(iteration, epoch);
    }

    if (train && init.isWeightParam(layer.conf().getLayer(), paramKey)
            || (applyToBiases && init.isBiasParam(layer.conf().getLayer(), paramKey))) {
        INDArray out = workspaceMgr.createUninitialized(ArrayType.INPUT, param.dataType(), param.shape(), param.ordering());
        Nd4j.getExecutioner().exec(new DropOut(param, out, p));
        return out;
    }
    return param;
}
 

public static INDArray reshape2dTo4d(INDArray in2d, long[] toShape, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(in2d.rank() != 2)
        throw new IllegalArgumentException("Invalid input: expect NDArray with rank 2");
    if (toShape.length != 4)
        throw new IllegalArgumentException("Invalid input: expect toShape with 4 elements: got " + Arrays.toString(toShape));

    if (in2d.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in2d))
        in2d = workspaceMgr.dup(type, in2d, 'c');

    if(format == CNN2DFormat.NCHW) {
        //Reshape: from [n*h*w,c] to [n,h,w,c] to [n,c,h,w]
        INDArray out = in2d.reshape('c', toShape[0], toShape[2], toShape[3], toShape[1]);
        return workspaceMgr.leverageTo(type, out.permute(0, 3, 1, 2));
    } else {
        //Reshape: from [n*h*w,c] to [n,h,w,c]
        return workspaceMgr.leverageTo(type, in2d.reshape('c', toShape));
    }
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray previous, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); // Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, new long[] { w.size(0), delta.size(0) }, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    // Normally we would clear weightNoiseParams here - but we want to reuse them
    // for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
    // INDArray delta = lossFunction.computeGradient(labels2d, preOut,
    // layerConf().getActivationFunction(), maskArray);
    INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

    Gradient gradient = new DefaultGradient();

    INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
    Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); // Equivalent to: weightGradView.assign(input.transpose().mmul(delta));
    gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

    if (hasBias()) {
        INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGradView, 0); // biasGradView is initialized/zeroed first in sum op
        gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
    return new Pair<>(gradient, delta);
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    //For RNN: need to sum up the score over each time step before returning.

    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = preOutput2d(false, workspaceMgr);

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
            lossFunction.computeScoreArray(getLabels2d(workspaceMgr, ArrayType.FF_WORKING_MEM), preOut,
                    layerConf().getActivationFn(), maskArray);
    INDArray summedScores = scoreArray.sum(1);

    if (fullNetRegTerm != 0.0) {
        summedScores.addi(fullNetRegTerm);
    }

    return summedScores;
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoBackward()) {
        throw new IllegalStateException("Cannot do backward pass: errors not set");
    }

    // epsilons[index] => {batchSize, numberOfEmbeds}
    INDArray[] epsilons = new INDArray[inputs.length];
    epsilons[0] = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, inputs[0]);
    epsilons[1] = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, inputs[1]);
    // epsilon => {batchSize, 1}
    // inputs[index] => {batchSize, numberOfEmbeds}
    // TODO 如何通过inputs[index]与epsilon求导epsilons[index]
    INDArray left = inputs[0];
    INDArray right = inputs[1];
    for (int index = 0; index < epsilon.rows(); index++) {
        epsilons[0].putRow(index, right.getRow(index).transpose().mmul(epsilon.getRow(index)).transpose());
        epsilons[1].putRow(index, left.getRow(index).transpose().mmul(epsilon.getRow(index)).transpose());
    }
    return new Pair<>(null, epsilons);
}
 

@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Epsilons are 2d, with shape [miniBatchSize, outChannels*outD*outH*outW]

    if (!hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    if (epsilons.rank() == 5)
        return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons); //Should never happen

    if (epsilons.columns() != inputDepth * inputWidth * inputHeight * numChannels)
        throw new IllegalArgumentException("Invalid input: expect output to have depth: "
                + inputDepth + ", height: " + inputHeight + ", width: " + inputWidth + " and channels: "
                + numChannels + ", i.e. [" + epsilons.rows() + ", "
                + inputDepth * inputHeight * inputWidth * numChannels + "] but was instead "
                + Arrays.toString(epsilons.shape()));

    INDArray ret;
    if (isNCDHW)
        ret = epsilons.reshape('c', epsilons.size(0), numChannels, inputDepth, inputHeight, inputWidth);
    else
        ret = epsilons.reshape('c', epsilons.size(0), inputDepth, inputHeight, inputWidth, numChannels);

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, ret); //Move to specified workspace if required

}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    // centers
    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray l = labels.castTo(centers.dataType());     //Ensure correct dtype (same as params); no-op if already correct dtype
    INDArray centersForExamples = l.mmul(centers);
    INDArray dLcdai = input.sub(centersForExamples);

    INDArray w = getParamWithNoise(CenterLossParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, w.dataType(), new long[]{w.size(0), delta.size(0)}, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();
    double lambda = layerConf().getLambda();
    epsilonNext.addi(dLcdai.muli(lambda)); // add center loss here

    weightNoiseParams.clear();

    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

/**Compute the score for each example individually, after labels and input have been set.
 *
 * @param fullNetRegTerm Regularization score term for the entire network (or, 0.0 to not include regularization)
 * @return A column INDArray of shape [numExamples,1], where entry i is the score of the ith example
 */
@Override
public INDArray computeScoreForExamples(double fullNetRegTerm, LayerWorkspaceMgr workspaceMgr) {
    if (input == null || labels == null)
        throw new IllegalStateException("Cannot calculate score without input and labels " + layerId());
    INDArray preOut = input;

    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray scoreArray =
                    lossFunction.computeScoreArray(getLabels2d(), preOut, layerConf().getActivationFn(), maskArray);
    if (fullNetRegTerm != 0.0) {
        scoreArray.addi(fullNetRegTerm);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, scoreArray);
}
 

/**
 * Reshape time series mask arrays. This should match the assumptions (f order, etc) in RnnOutputLayer
 * @param timeSeriesMask    Mask array to reshape to a column vector
 * @return                  Mask array as a column vector
 */
public static INDArray reshapeTimeSeriesMaskToVector(INDArray timeSeriesMask, LayerWorkspaceMgr workspaceMgr, ArrayType arrayType) {
    if (timeSeriesMask.rank() != 2)
        throw new IllegalArgumentException("Cannot reshape mask: rank is not 2");

    if (timeSeriesMask.ordering() != 'f' || !Shape.hasDefaultStridesForShape(timeSeriesMask))
        timeSeriesMask = workspaceMgr.dup(arrayType, timeSeriesMask, 'f');

    return workspaceMgr.leverageTo(arrayType, timeSeriesMask.reshape('f', timeSeriesMask.length(), 1));
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoBackward())
        throw new IllegalStateException("Cannot do backward pass: errors not set");

    INDArray[] out = new INDArray[1];
    out[0] = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilon.reshape(order, inputs[0].shape()));
    return new Pair<>(null, out);
}
 

@Override
public INDArray doForward(boolean training, LayerWorkspaceMgr workspaceMgr) {
    if (!canDoForward())
        throw new IllegalStateException("Cannot do forward pass: input not set");

    // once we know the inputs, save the shape and interval size for doBackward
    this.forwardShape = Arrays.copyOf(inputs[0].shape(), inputs[0].rank());

    this.step = inputs[0].size(0) / stackSize;
    long start = from * step;
    long end = (from + 1) * step;

    INDArray ret;
    switch (inputs[0].rank()) { //TODO remove the dups here if/when possible (gradient checks must pass)
        case 2:
            ret = inputs[0].get(NDArrayIndex.interval(start, end), NDArrayIndex.all());
            break;
        case 3:
            ret = inputs[0].get(NDArrayIndex.interval(start, end), NDArrayIndex.all(), NDArrayIndex.all());
            break;
        case 4:
            ret = inputs[0].get(NDArrayIndex.interval(start, end), NDArrayIndex.all(), NDArrayIndex.all(),
                            NDArrayIndex.all());
            break;
        default:
            throw new UnsupportedOperationException(
                            "Cannot get subset for activations of rank " + inputs[0].rank());
    }

    return workspaceMgr.dup(ArrayType.ACTIVATIONS, ret);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    INDArray temp = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, input, input.ordering());
    INDArray delta = layerConf().getActivationFn().backprop(temp, epsilon).getFirst(); //TODO handle activation function params
    if(delta == epsilon ){
        //Edge case: identity activation + external errors -> no-op
        delta = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, delta);
    }

    delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);  //Usually a no-op (except for perhaps identity)
    Gradient ret = new DefaultGradient();
    return new Pair<>(ret, delta);
}
 

@Override
public INDArray activate(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    boolean nchw = layerConf().getDataFormat() == CNN2DFormat.NCHW;
    int hIdx = nchw ? 2 : 1;
    int wIdx = nchw ? 3 : 2;

    int[] padding = layerConf().getPadding();
    val inShape = input.shape();
    val outH = inShape[hIdx] + padding[0] + padding[1];
    val outW = inShape[wIdx] + padding[2] + padding[3];
    val outShape = nchw ? new long[] {inShape[0], inShape[1], outH, outW} : new long[] {inShape[0], outH, outW, inShape[3]};

    INDArray out = workspaceMgr.create(ArrayType.ACTIVATIONS, input.dataType(), outShape, 'c');

    if(nchw) {
        out.put(new INDArrayIndex[]{NDArrayIndex.all(), NDArrayIndex.all(),
                NDArrayIndex.interval(padding[0], padding[0] + inShape[hIdx]),
                NDArrayIndex.interval(padding[2], padding[2] + inShape[wIdx])}, input);
    } else {
        out.put(new INDArrayIndex[]{NDArrayIndex.all(),
                NDArrayIndex.interval(padding[0], padding[0] + inShape[hIdx]),
                NDArrayIndex.interval(padding[2], padding[2] + inShape[wIdx]),
                NDArrayIndex.all()}, input);
    }

    return out;
}
 

@Override
public Pair<Gradient, INDArray[]> doBackward(boolean tbptt, LayerWorkspaceMgr workspaceMgr) {

    // Get the mask arrays for the given input, if any
    INDArray mask = getMask();

    // Backpropagate the output error (epsilon) to the input variables:
    //      Just undo the revert (which can be done by another revert)
    INDArray epsilonsOut = revertTimeSeries(epsilon, mask, workspaceMgr, ArrayType.ACTIVATION_GRAD);

    return new Pair<>(null, new INDArray[] {epsilonsOut});
}
 

@Override
public INDArray activate(INDArray input, boolean training, LayerWorkspaceMgr workspaceMgr) {
    INDArray reshaped = reshape(input);
    INDArray out = underlying.activate(reshaped, training, workspaceMgr);
    INDArray ret = revertReshape(out, input.size(0));
    return workspaceMgr.dup(ArrayType.ACTIVATIONS, ret);
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Apply input preprocessors in opposite order for backprop (compared to forward pass)
    //For example, CNNtoFF + FFtoRNN, need to do backprop in order of FFtoRNN + CNNtoFF
    for (int i = inputPreProcessors.length - 1; i >= 0; i--) {
        output = inputPreProcessors[i].backprop(output, miniBatchSize, workspaceMgr);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    if (input.rank() != 5)
        throw new UnsupportedOperationException(
                "Input is not rank 5. Got input with rank " + input.rank() + " " + layerId() + " with shape "
                        + Arrays.toString(input.shape()) + " - expected shape [minibatch,channels,depth,height,width]");
    if (labels == null)
        throw new IllegalStateException("Labels are not set (null)");

    INDArray input2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), input, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray labels2d = ConvolutionUtils.reshape5dTo2d(layerConf().getDataFormat(), labels, workspaceMgr, ArrayType.FF_WORKING_MEM);
    INDArray maskReshaped = ConvolutionUtils.reshapeCnn3dMask(layerConf().getDataFormat(), maskArray, labels, workspaceMgr, ArrayType.FF_WORKING_MEM);

    // delta calculation
    ILossFunction lossFunction = layerConf().getLossFn();
    INDArray delta2d = lossFunction.computeGradient(labels2d, input2d.dup(input2d.ordering()), layerConf().getActivationFn(), maskReshaped);
    delta2d = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta2d);

    long n = input.size(0);
    long d, h, w, c;
    if(layerConf().getDataFormat() == Convolution3D.DataFormat.NDHWC){
        d = input.size(1);
        h = input.size(2);
        w = input.size(3);
        c = input.size(4);
    } else {
        d = input.size(2);
        h = input.size(3);
        w = input.size(4);
        c = input.size(1);
    }
    INDArray delta5d = ConvolutionUtils.reshape2dTo5d(layerConf().getDataFormat(), delta2d, n, d, h, w, c, workspaceMgr, ArrayType.ACTIVATION_GRAD);

    // grab the empty gradient
    Gradient gradient = new DefaultGradient();
    return new Pair<>(gradient, delta5d);
}