Java Code Examples for org.nd4j.linalg.api.shape.Shape#hasDefaultStridesForShape()

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (input.ordering() != 'f' || !Shape.hasDefaultStridesForShape(input))
        input = input.dup('f');
    //Input: 3d activations (RNN)
    //Output: 4d activations (CNN)
    if (rnnDataFormat == RNNFormat.NWC){
        input = input.permute(0, 2, 1);
    }
    val shape = input.shape();
    INDArray in2d;
    if (shape[0] == 1) {
        //Edge case: miniBatchSize = 1
        in2d = input.tensorAlongDimension(0, 1, 2).permutei(1, 0);
    } else if (shape[2] == 1) {
        //Edge case: time series length = 1
        in2d = input.tensorAlongDimension(0, 1, 0);
    } else {
        INDArray permuted = input.permute(0, 2, 1); //Permute, so we get correct order after reshaping
        in2d = permuted.reshape('f', shape[0] * shape[2], shape[1]);
    }

    return workspaceMgr.dup(ArrayType.ACTIVATIONS, in2d, 'c')
            .reshape('c', shape[0] * shape[2], numChannels, inputHeight, inputWidth);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    this.shape = input.shape();
    if (input.rank() == 4)
        return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input);

    if (input.columns() != inputWidth * inputHeight * numChannels)
        throw new IllegalArgumentException("Invalid input: expect output columns must be equal to rows "
                + inputHeight + " x columns " + inputWidth + " x channels " + numChannels
                + " but was instead " + Arrays.toString(input.shape()));

    if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input))
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');

    return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS,
            input.reshape('c', input.size(0), numChannels, inputHeight, inputWidth));
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    long[] targetShape = getShape(this.targetShape, miniBatchSize);
    long[] inputShape = getShape(this.inputShape, miniBatchSize);

    if (!Arrays.equals(targetShape, output.shape())) {
        throw new IllegalStateException("Unexpected output shape" + Arrays.toString(output.shape())
                + " (expected to be " + Arrays.toString(targetShape) + ")");
    }
    if (prodLong(output.shape()) == prodLong((targetShape))) {
        if (output.ordering() != 'c' || !Shape.hasDefaultStridesForShape(output)) {
            output = workspaceMgr.dup(ArrayType.ACTIVATIONS, output, 'c');
        }
        return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, output.reshape(inputShape));
    } else {
        throw new IllegalStateException("Output shape" + Arrays.toString(output.shape())
                + " and input shape" + Arrays.toString(targetShape) + " do not match");
    }
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Input: 4d epsilons (CNN)
    //Output: 3d epsilons (RNN)
    if (output.ordering() != 'c' || !Shape.hasDefaultStridesForShape(output))
        output = output.dup('c');
    val shape = output.shape();
    //First: reshape 4d to 2d
    INDArray twod = output.reshape('c', output.size(0), ArrayUtil.prod(output.shape()) / output.size(0));
    //Second: reshape 2d to 3d
    INDArray reshaped = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, twod, 'f').reshape('f', miniBatchSize, shape[0] / miniBatchSize, product);
    if (rnnDataFormat == RNNFormat.NCW) {
        reshaped = reshaped.permute(0, 2, 1);
    }
    return reshaped;
}
 

public static INDArray reshape2dTo4d(INDArray in2d, long[] toShape, CNN2DFormat format, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    if(in2d.rank() != 2)
        throw new IllegalArgumentException("Invalid input: expect NDArray with rank 2");
    if (toShape.length != 4)
        throw new IllegalArgumentException("Invalid input: expect toShape with 4 elements: got " + Arrays.toString(toShape));

    if (in2d.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in2d))
        in2d = workspaceMgr.dup(type, in2d, 'c');

    if(format == CNN2DFormat.NCHW) {
        //Reshape: from [n*h*w,c] to [n,h,w,c] to [n,c,h,w]
        INDArray out = in2d.reshape('c', toShape[0], toShape[2], toShape[3], toShape[1]);
        return workspaceMgr.leverageTo(type, out.permute(0, 3, 1, 2));
    } else {
        //Reshape: from [n*h*w,c] to [n,h,w,c]
        return workspaceMgr.leverageTo(type, in2d.reshape('c', toShape));
    }
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    //Need to reshape RNN epsilons (3d) to 2d (for use in FF layer backprop calculations)
    if (output.rank() != 3)
        throw new IllegalArgumentException(
                        "Invalid input: expect NDArray with rank 3 (i.e., epsilons from RNN layer)");
    if (output.ordering() != 'f' || !Shape.hasDefaultStridesForShape(output))
        output = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, output, 'f');
    if (rnnDataFormat == RNNFormat.NWC){
        output = output.permute(0, 2, 1);
    }
    val shape = output.shape();

    INDArray ret;
    if (shape[0] == 1) {
        ret = output.tensorAlongDimension(0, 1, 2).permutei(1, 0); //Edge case: miniBatchSize==1
    } else if (shape[2] == 1) {
        return output.tensorAlongDimension(0, 1, 0); //Edge case: timeSeriesLength=1
    } else {
        INDArray permuted = output.permute(0, 2, 1); //Permute, so we get correct order after reshaping
        ret = permuted.reshape('f', shape[0] * shape[2], shape[1]);
    }
    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, ret);
}
 

/**
 * Given a mask array for a 1D CNN layer of shape [minibatch, sequenceLength], reduce the mask according to the 1D CNN layer configuration.
 * Unlike RNN layers, 1D CNN layers may down-sample the data; consequently, we need to down-sample the mask array
 * in the same way, to maintain the correspondence between the masks and the output activations
 *
 * @param in       Input size
 * @param kernel   Kernel size
 * @param stride   Stride
 * @param padding  Padding
 * @param dilation Dilation
 * @param cm       Convolution mode
 * @return Reduced mask
 */
public static INDArray cnn1dMaskReduction(INDArray in, int kernel, int stride, int padding, int dilation, ConvolutionMode cm){
    Preconditions.checkState(in.rank()==2, "Rank must be 2 for cnn1d mask array - shape ", in.shape());
    if((cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal) && stride == 1 ){
        return in;
    }

    if(!Shape.hasDefaultStridesForShape(in)){
        in = in.dup();
    }

    INDArray reshaped4d = in.reshape(in.size(0), 1, in.size(1), 1);

    int[] outSize;
    int[] pad = null;
    int[] k = new int[]{kernel,1};
    int[] s = new int[]{stride, 1};
    int[] d = new int[]{dilation, 1};
    if (cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal) {
        outSize = ConvolutionUtils.getOutputSize(reshaped4d, k, s, null, cm, d, CNN2DFormat.NCHW); //Also performs validation
    } else {
        pad = new int[]{padding, 0};
        outSize = ConvolutionUtils.getOutputSize(reshaped4d, k, s, pad, cm, d, CNN2DFormat.NCHW); //Also performs validation
    }
    int outH = outSize[0];

    INDArray output = Nd4j.createUninitialized(new int[]{(int)in.size(0), 1, outH, 1}, 'c');

    DynamicCustomOp op = new MaxPooling2D(reshaped4d, output, Pooling2DConfig.builder()
            .kH(k[0]).kW(k[1])
            .sH(s[0]).sW(s[1])
            .pH(pad == null ? 0 : pad[0]).pW(pad == null ? 0 : pad[1])
            .dH(d[0]).dW(d[1])
            .isSameMode(cm == ConvolutionMode.Same || cm == ConvolutionMode.Causal)
            .isNHWC(false)
            .build());

    Nd4j.getExecutioner().exec(op);
    return output.reshape('c', in.size(0), outH);
}
 

@Override
// return 4 dimensions
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (epsilons.ordering() != 'c' || !Shape.hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    if (shape == null || ArrayUtil.prod(shape) != epsilons.length()) {
        if (epsilons.rank() == 2)
            return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons); //should never happen

        return epsilons.reshape('c', epsilons.size(0), numChannels, inputHeight, inputWidth);
    }

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons.reshape('c', shape));
}
 

@Override
public INDArray ravel(char ordering) {
    Nd4j.getCompressor().autoDecompress(this);
    if(ordering == this.ordering() && Shape.hasDefaultStridesForShape(this)){
        return reshape(ordering, length());
    }
    return dup(ordering).reshape(ordering, length());
}
 

@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (epsilons.ordering() != 'c' || !Shape.hasDefaultStridesForShape(epsilons))
        epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');

    INDArray epsilonsReshaped = epsilons.reshape('c', epsilons.size(0), inputHeight, inputWidth, numChannels);

    return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilonsReshaped.permute(0, 3, 1, 2));    //To [n, c, h, w]
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (permutationIndices.length + 1 == input.shape().length) {
        permutationIndices = prependZero(permutationIndices);
        this.hasLeadingDimension = true;
    }
    if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input)) {
        input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');
    }
    INDArray output = workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, input.permute(this.permutationIndices));
    return output;
}
 

public SerializedNDArray convert(INDArray from){
    if(from.isView() || from.ordering() != 'c' || !Shape.hasDefaultStridesForShape(from))
        from = from.dup('c');

    NDArrayType type = ND4JUtil.typeNd4jToNDArrayType(from.dataType());
    long[] shape = from.shape();
    ByteBuffer bb = from.data().asNio();

    return new SerializedNDArray(type, shape, bb);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);

    INDArray input = this.input.castTo(epsilon.dataType());

    boolean nchw = layerConf().getDataFormat() == CNN2DFormat.NCHW;
    long miniBatch = input.size(0);
    long inDepth = input.size(nchw ? 1 : 3);
    long inH = input.size(nchw ? 2 : 1);
    long inW = input.size(nchw ? 3 : 2);

    long[] epsShape = nchw ?  new long[]{miniBatch, inDepth, inH, inW} : new long[]{miniBatch, inH, inW, inDepth};
    INDArray outEpsilon = workspaceMgr.create(ArrayType.ACTIVATION_GRAD, input.dataType(), epsShape, 'c');

    Gradient gradient = new DefaultGradient();

    int blockSize = getBlockSize();

    //Workaround for issue: https://github.com/eclipse/deeplearning4j/issues/8859
    if(!Shape.hasDefaultStridesForShape(epsilon))
        epsilon = epsilon.dup('c');

    CustomOp op = DynamicCustomOp.builder("depth_to_space")
            .addInputs(epsilon)
            .addIntegerArguments(blockSize, nchw ? 0 : 1)       //nchw = 0, nhwc = 1
            .addOutputs(outEpsilon)
            .build();
    Nd4j.getExecutioner().exec(op);

    return new Pair<>(gradient, outEpsilon);
}
 

@Override
public INDArray backprop(INDArray output, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
    if (output.ordering() == 'c' || !Shape.hasDefaultStridesForShape(output))
        output = output.dup('f');
    if (rnnDataFormat == RNNFormat.NWC){
        output = output.permute(0, 2, 1);
    }
    val shape = output.shape();
    INDArray output2d;
    if (shape[0] == 1) {
        //Edge case: miniBatchSize = 1
        output2d = output.tensorAlongDimension(0, 1, 2).permutei(1, 0);
    } else if (shape[2] == 1) {
        //Edge case: timeSeriesLength = 1
        output2d = output.tensorAlongDimension(0, 1, 0);
    } else {
        //As per FeedForwardToRnnPreprocessor
        INDArray permuted3d = output.permute(0, 2, 1);
        output2d = permuted3d.reshape('f', shape[0] * shape[2], shape[1]);
    }

    if (shape[1] != product)
        throw new IllegalArgumentException("Invalid input: expected output size(1)=" + shape[1]
                        + " must be equal to " + inputHeight + " x columns " + inputWidth + " x channels "
                        + numChannels + " = " + product + ", received: " + shape[1]);
    INDArray ret = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, output2d, 'c');
    return ret.reshape('c', output2d.size(0), numChannels, inputHeight, inputWidth);
}
 

public static INDArray reshape5dTo2d(@NonNull Convolution3D.DataFormat format, INDArray in, LayerWorkspaceMgr workspaceMgr, ArrayType type){
    Preconditions.checkState(in.rank() == 5, "Invalid input: expect NDArray with rank 5, got rank %ndRank with shape %ndShape", in, in);
    //Reshape: from either [n,c,d,h,w] to [n*d*h*w,c] (NCDHW format)
    // or reshape from [n,d,h,w,c] to [n*d*h*w,c] (NDHWC format)
    if(format != Convolution3D.DataFormat.NDHWC){
        in = in.permute(0, 2, 3, 4, 1);
    }

    if(in.ordering() != 'c' || !Shape.hasDefaultStridesForShape(in))
        in = workspaceMgr.dup(type, in, 'c');
    return workspaceMgr.leverageTo(type, in.reshape('c', in.size(0)*in.size(1)*in.size(2)*in.size(3), in.size(4)));
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray input, INDArray epsilon, long[] shape, INDArray gamma,
                                                 INDArray beta, INDArray dGammaView, INDArray dBetaView, double eps,
                                                 CNN2DFormat format, LayerWorkspaceMgr workspaceMgr) {

    //Workaround for: https://github.com/eclipse/deeplearning4j/issues/8860
    if(!Shape.hasDefaultStridesForShape(epsilon))
        epsilon = epsilon.dup('c');

    if(input.dataType() != DataType.FLOAT)
        return null;    //MKL-DNN only supports float

    int axis = (input.rank() != 4 || format == CNN2DFormat.NCHW) ? 1 : 3;

    List<INDArray> args = new ArrayList<>();
    args.add(input);
    args.add(meanCache);
    args.add(varCache);
    if(gamma != null)
        args.add(gamma.reshape(gamma.length()));
    if(beta != null)
        args.add(beta.reshape(beta.length()));
    args.add(epsilon);


    DynamicCustomOp op = DynamicCustomOp.builder("batchnorm_bp")
            .addInputs(args.toArray(new INDArray[0]))
            .addIntegerArguments(
                    gamma == null ? 0 : 1,          //Apply scale
                    beta == null ? 0 : 1,           //Apply beta
                    axis)                              //Axis (NCHW) - 1=NCHW, 3=NHWC
            .addFloatingPointArguments(eps)
            .build();

    INDArray epsAtInput = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), input.shape());
    INDArray dLdm = workspaceMgr.createUninitialized(ArrayType.BP_WORKING_MEM, meanCache.dataType(), meanCache.shape());
    INDArray dLdv = workspaceMgr.createUninitialized(ArrayType.BP_WORKING_MEM, meanCache.dataType(), meanCache.shape());

    op.setOutputArgument(0, epsAtInput);
    op.setOutputArgument(1, dLdm);
    op.setOutputArgument(2, dLdv);
    if(dGammaView != null) {
        //Both are always null/not null simultaneously
        op.setOutputArgument(3, dGammaView.reshape(dGammaView.length()));
        op.setOutputArgument(4, dBetaView.reshape(dBetaView.length()));
    }


    Nd4j.exec(op);

    Gradient g = new DefaultGradient();
    g.setGradientFor(BatchNormalizationParamInitializer.GAMMA, dGammaView);
    g.setGradientFor(BatchNormalizationParamInitializer.BETA, dBetaView);

    return new Pair<>(g, epsAtInput);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray input, INDArray epsilon, long[] shape, INDArray gamma, INDArray beta,
                                                 INDArray dGammaView, INDArray dBetaView, double eps, CNN2DFormat format, LayerWorkspaceMgr layerWorkspaceMgr) {

    boolean nchw = format == CNN2DFormat.NCHW;

    this.eps = eps;

    int cudnnTensorFormat = nchw ? CUDNN_TENSOR_NCHW : CUDNN_TENSOR_NHWC;
    int chIdx = nchw ? 1 : 3;
    int hIdx = nchw ? 2 : 1;
    int wIdx = nchw ? 3 : 2;

    val miniBatch = (int) input.size(0);
    val depth = (int) input.size(chIdx);
    val inH = (int) input.size(hIdx);
    val inW = (int) input.size(wIdx);

    final boolean isHalf = (input.dataType() == DataType.HALF);
    INDArray gammaOrig = null;
    INDArray dGammaViewOrig = null;
    INDArray dBetaViewOrig = null;
    if(isHalf) {    //Convert FP16 to FP32 if required (CuDNN BN doesn't support FP16 for these params, only for input/output)
        gammaOrig = gamma;
        dGammaViewOrig = dGammaView;
        dBetaViewOrig = dBetaView;
        /*
        From CuDNN docs: bnScale, resultBnScaleDiff, resultBnBiasDiff, savedMean, savedInvVariance
        "Note: The data type of this tensor descriptor must be 'float' for FP16 and FP32 input tensors, and 'double'
        for FP64 input tensors."
        >> Last 2 are the meanCache and varCache; first 3 are below
         */
        gamma = gamma.castTo(DataType.FLOAT);
        dGammaView = dGammaView.castTo(DataType.FLOAT);
        dBetaView = dBetaView.castTo(DataType.FLOAT);
    }

    Gradient retGradient = new DefaultGradient();

    if (!Shape.hasDefaultStridesForShape(epsilon)) {
        // apparently not supported by cuDNN
        epsilon = epsilon.dup('c');
    }

    val srcStride = ArrayUtil.toInts(input.stride());
    val deltaStride = ArrayUtil.toInts(epsilon.stride());

    if (Nd4j.getExecutioner() instanceof GridExecutioner)
        ((GridExecutioner) Nd4j.getExecutioner()).flushQueue();

    checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.srcTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW,
            (int) srcStride[0], (int) srcStride[chIdx], (int) srcStride[hIdx], (int) srcStride[wIdx]));
    checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.deltaTensorDesc, dataType, (int) miniBatch, (int) depth, (int) inH, (int) inW,
            (int) deltaStride[0], (int) deltaStride[chIdx], (int) deltaStride[hIdx], (int) deltaStride[wIdx]));

    long[] nextEpsShape = nchw ? new long[] {miniBatch, depth, inH, inW} : new long[] {miniBatch, inH, inW, depth};
    INDArray nextEpsilon = layerWorkspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, input.dataType(), nextEpsShape, 'c');
    val dstStride = ArrayUtil.toInts(nextEpsilon.stride());

    checkCudnn(cudnnSetTensor4dDescriptorEx(cudnnContext.dstTensorDesc, dataType, miniBatch, depth, inH, inW,
                    dstStride[0], dstStride[chIdx], dstStride[hIdx], dstStride[wIdx]));
    checkCudnn(cudnnSetTensor4dDescriptor(cudnnContext.gammaBetaTensorDesc, cudnnTensorFormat, toCudnnDataType(gamma.data().dataType()), (int)shape[0],
            (int)shape[1], shape.length > 2 ? (int)shape[2] : 1, shape.length > 3 ? (int)shape[3] : 1));

    Allocator allocator = AtomicAllocator.getInstance();
    CudaContext context = allocator.getFlowController().prepareActionAllWrite(input, epsilon, nextEpsilon, gamma,
                    dGammaView, dBetaView);
    Pointer srcData = allocator.getPointer(input, context);
    Pointer epsData = allocator.getPointer(epsilon, context);
    Pointer dstData = allocator.getPointer(nextEpsilon, context);
    Pointer gammaData = allocator.getPointer(gamma, context);
    Pointer dGammaData = allocator.getPointer(dGammaView, context);
    Pointer dBetaData = allocator.getPointer(dBetaView, context);
    Pointer meanCacheData = allocator.getPointer(meanCache, context);
    Pointer varCacheData = allocator.getPointer(varCache, context);

    checkCudnn(cudnnSetStream(cudnnContext, new CUstream_st(context.getCublasStream())));
    checkCudnn(cudnnBatchNormalizationBackward(cudnnContext, batchNormMode, alpha, this.beta, alpha, alpha,
                    cudnnContext.srcTensorDesc, srcData, cudnnContext.deltaTensorDesc, epsData,
                    cudnnContext.dstTensorDesc, dstData, cudnnContext.gammaBetaTensorDesc, gammaData, dGammaData,
                    dBetaData, eps, meanCacheData, varCacheData));

    allocator.getFlowController().registerActionAllWrite(context, input, epsilon, nextEpsilon, gamma, dGammaView,
                    dBetaView);

    retGradient.setGradientFor(BatchNormalizationParamInitializer.GAMMA, dGammaView);
    retGradient.setGradientFor(BatchNormalizationParamInitializer.BETA, dBetaView);

    context.syncOldStream();

    //Convert back and assign, if required:
    if(isHalf){
        gammaOrig.assign(gamma.castTo(DataType.HALF));
        dGammaViewOrig.assign(dGammaView.castTo(DataType.HALF));
        dBetaViewOrig.assign(dBetaView.castTo(DataType.HALF));
    }

    return new Pair<>(retGradient, nextEpsilon);
}
 

@Override
public INDArray mmuli(INDArray other, INDArray result) {
    validateNumericalArray("mmuli", false);
    LinAlgExceptions.assertMultiplies(this, other);
    if(other.rank() == 1){
        //GEMV edge case
        Preconditions.checkState(result.length() == this.size(0) && this.size(1) == other.size(0),
                "Invalid matrix multiplication: %ndShape x %ndShape with result shape %ndShape", this, other, result);
    } else {
        //Standard case
        Preconditions.checkState(
                result.rank() == 2 && result.size(0) == this.size(0) && result.size(1) == other.size(1),
                "Invalid result array shape: expected shape [%s,%s], got shape %ndShape result array for %ndShape x %ndShape", this.size(0), other.size(1), result,
                this, other);
    }

    if (other.isScalar()) {
        return muli(other.getDouble(0), result);
    }
    if (isScalar()) {
        return other.muli(getDouble(0), result);
    }

    /* check sizes and resize if necessary */


    if (result == this || result == other) {
        /* actually, blas cannot do multiplications in-place. Therefore, we will fake by
         * allocating a temporary object on the side and copy the result later.
         */
        INDArray temp = Nd4j.create(result.dataType(), result.shape(), Nd4j.getStrides(result.shape(), 'f'), 'f');

        if (other.columns() == 1 || other.rank() == 1) {
            Nd4j.getBlasWrapper().level2().gemv(BlasBufferUtil.getCharForTranspose(result),
                    BlasBufferUtil.getCharForTranspose(this), 1.0, this, other, 0.0, temp);
        }

        else {
            Nd4j.getBlasWrapper().level3().gemm(BlasBufferUtil.getCharForTranspose(result),
                    BlasBufferUtil.getCharForTranspose(this), BlasBufferUtil.getCharForTranspose(temp), 1.0,
                    this, other, 0.0, temp);
        }

        result.assign(temp);


    } else {

        //We require that the result array is 'f' (fortran) order
        // However, user might have called mmuli with a c order array for the result
        // In which case, we need to allocate a temporary f order array, and later do an assign to the real result array

        boolean requiresTemp = result.ordering() != 'f' || result.isView() || !Shape.hasDefaultStridesForShape(result);
        INDArray gemmResultArr;
        if (requiresTemp) {
            //Can use createUninitialized due to beta==0.0 parameter in gemm
            gemmResultArr = Nd4j.createUninitialized(result.dataType(), result.shape(), 'f');
        } else {
            gemmResultArr = result;
        }

        if (other.columns() == 1 || other.rank() == 1) {
            Nd4j.getBlasWrapper().level2().gemv(
                    ordering(),
                    BlasBufferUtil.getCharForTranspose(other),
                    1.0,
                    this,
                    other,
                    0.0,
                    gemmResultArr);
        } else {
            //gemm doesn't support strides so vectors and views
            //don't work
            Nd4j.getBlasWrapper().level3().gemm(ordering(),
                    BlasBufferUtil.getCharForTranspose(other),
                    BlasBufferUtil.getCharForTranspose(gemmResultArr),
                    1.0,
                    this,
                    other,
                    0.0,
                    gemmResultArr);
        }

        if (requiresTemp) {
            result.assign(gemmResultArr);
        }
    }

    // 1D edge case: reshape back to vector
    if (other.rank() == 1)
        result = result.reshape(result.length());
    return result;
}
 

/**
 * Reverse the preProcess during backprop. Process Gradient/epsilons before passing them to the layer below.
 * 
 * @param epsilons which is a pair of the gradient and epsilon. 后一层的epsilon是目标函数对前一层的激活函数值的偏导。要是为0的话 会导致前一层的权重的梯度为0。
 * @param miniBatchSize
 * @param workspaceMgr
 * @return the reverse of the pre preProcess step (if any). Note that the returned array should be placed in
 *         {@link ArrayType#ACTIVATION_GRAD} workspace via the workspace manager
 */
@Override
public INDArray backprop(INDArray epsilons, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
	if (printLog) {
		System.out.println("this.isRInputEmpty = " + this.isRInputEmpty);
	}
	// System.out.println("epsilons.sumNumber() = " + epsilons.sumNumber());
	// System.out.println("epsilons.shape() = " + Arrays.toString(epsilons.shape()));
	if (epsilons.ordering() != 'c' || !Shape.hasDefaultStridesForShape(epsilons)) {
		epsilons = workspaceMgr.dup(ArrayType.ACTIVATION_GRAD, epsilons, 'c');
	}

	if (shape == null || ArrayUtil.prod(shape) != epsilons.length()) {
		INDArray newEpsilons = Nd4j.zeros(shape[0], shape[1], shape[2], shape[3]);
		for (int i = 0; i < shape[0]; i++) {
			// [numChannels * 1, inputHeight, inputWidth]
			INDArray multyImage = epsilons.getRow(i);
			// [numChannels * 2, inputHeight, inputWidth]
			INDArray newMultyImage = newEpsilons.getRow(i);

			// INDArray zeroINDArray = Nd4j.zeros(shape[3], shape[2]);

			// System.out.println("shape[1] = " + shape[1]);
			for (int j = 0; j < shape[1] / 2; j++) {
				// [inputHeight, inputWidth]
				INDArray imageWH = multyImage.getRow(j);
				// System.out.println("imageWH = " + imageWH);
				if (printLog) {
					System.out.println("imageWH.shape() = " + Arrays.toString(imageWH.shape()));
				}
				if (this.isRInputEmpty) {
					newMultyImage.putRow(j, imageWH);
					// 其他的默认是0值
					// newMultyImage.putRow(j + shape[1] / 2, zeroINDArray);
				} else {
					// newMultyImage.putRow(j, zeroINDArray);
					// 其他的默认是0值
					newMultyImage.putRow(j + shape[1] / 2, imageWH);
				}
			}
		}
		if (printLog) {
			System.out.println("newEpsilons.shape() = " + Arrays.toString(newEpsilons.shape()));
			System.out.println("newEpsilons = " + newEpsilons);
		}

		return newEpsilons;
		// return newEpsilons.reshape('c', newEpsilons.size(0), numChannels * 2, inputHeight, inputWidth);
	}

	if (printLog) {
		System.out.println("reshape to " + shape[1] * 2);
	}
	// return workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, epsilons.reshape('c', shape[0], shape[1] * 2, shape[2], shape[3]));
	return epsilons;
	// return epsilons.reshape('c', shape[0], shape[1] * 2, shape[2], shape[3]);
}
 

@Override
public INDArray preProcess(INDArray input, int miniBatchSize, LayerWorkspaceMgr workspaceMgr) {
	// [1 , numChannels * 2, inputHeight, inputWidth]
	this.shape = input.shape();
	// System.out.println("input = " + input);
	// System.out.println("input.sumNumber() = " + input.sumNumber());
	if (printLog) {
		System.out.println("this.shape = " + Arrays.toString(this.shape));
	}
	// Input: 4d activations (CNN)
	// Output: 4d activations (CNN)
	if (input.rank() != 4) {
		throw new IllegalArgumentException(
				"Invalid input: expect CNN activations with rank 4 (received input with shape " + Arrays.toString(input.shape()) + ")");
	}

	if (input.ordering() != 'c' || !Shape.hasDefaultStridesForShape(input)) {
		input = input.dup('c');
		// input = workspaceMgr.dup(ArrayType.ACTIVATIONS, input, 'c');
	}

	// 将2张CNN转为1张CNN
	INDArray newInput = Nd4j.zeros(shape[0], shape[1] / 2, shape[2], shape[3]);
	for (int i = 0; i < shape[0]; i++) {
		// [numChannels * 2, inputHeight, inputWidth]: z + r
		INDArray multyImage = input.get(NDArrayIndex.point(i), NDArrayIndex.all());
		// System.out.println("multyImage.sumNumber() = " + multyImage.sumNumber());
		// [numChannels * 1, inputHeight, inputWidth]
		INDArray newMultyImage = newInput.getRow(i);

		int newRowIndex = 0;
		for (int j = 0; j < shape[1] / 2; j++) {
			// [inputHeight, inputWidth]
			INDArray rImageWH = null;
			if (j == 0) {
				// 第一步，读取rImageWH，并判断它是否为空
				// "z-input", "r-input"
				rImageWH = multyImage.get(NDArrayIndex.point(j + shape[1] / 2), NDArrayIndex.all());
				// System.out.println("rImageWH.sumNumber() = " + rImageWH.sumNumber());
				double firstPixelValue = rImageWH.getDouble(0, 0);
				if (firstPixelValue != -9999) {
					this.isRInputEmpty = false;
				} else {
					this.isRInputEmpty = true;
				}
			}

			if (!this.isRInputEmpty) {
				if (rImageWH == null) {
					rImageWH = multyImage.get(NDArrayIndex.point(j + shape[1] / 2), NDArrayIndex.all());
				}
				// System.out.println("newRowIndex = " + newRowIndex);
				newMultyImage.putRow(newRowIndex, rImageWH);
				// System.out.println("newMultyImage.sumNumber() = " + newMultyImage.sumNumber());
			} else {
				INDArray zImageWH = multyImage.get(NDArrayIndex.point(j), NDArrayIndex.all());
				newMultyImage.putRow(newRowIndex, zImageWH);
			}
			newRowIndex++;
		}

		newInput.putRow(i, newMultyImage);
	}
	// System.out.println("newInput = " + newInput);
	// System.out.println("newInput.sumNumber() = " + newInput.sumNumber());

	// return workspaceMgr.leverageTo(ArrayType.ACTIVATIONS, newInput);
	if (save) {
		ImageUtils.save("/myself/tmp/dl4j/gan/data/train/0/0.jpg", newInput.dup().mul(255));
	}
	return newInput;
}