Java Code Examples for org.deeplearning4j.nn.api.Layer#activate()

@Test
public void testFeatureMapShapeMNIST() throws Exception {
    int inputWidth = 28;
    int[] stride = new int[] {1, 1};
    int[] padding = new int[] {0, 0};
    int[] kernelSize = new int[] {9, 9};
    int nChannelsIn = 1;
    int depth = 20;
    int featureMapWidth = (inputWidth + padding[1] * 2 - kernelSize[1]) / stride[1] + 1;

    INDArray input = getMnistData();

    Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding);
    INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(featureMapWidth, convActivations.size(2));
    assertEquals(depth, convActivations.size(1));
}
 

@Test
public void testSubSampleLayerAvgBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 2., 3., 4., 5., 6., 7., 8.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0.25, 0.25, 0.5, 0.5, 0.25, 0.25, 0.5, 0.5,
                    0.75, 0.75, 1., 1., 0.75, 0.75, 1., 1., 1.25, 1.25, 1.5, 1.5, 1.25, 1.25, 1.5, 1.5, 1.75, 1.75,
                    2., 2., 1.75, 1.75, 2., 2.}, new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());
    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertArrayEquals(expectedContainedEpsilonResult.shape(), containedOutput.getSecond().shape());

}
 

@Test
public void testSubSampleMeanActivate() throws Exception {
    INDArray containedExpectedOut =
                    Nd4j.create(new double[] {2., 4., 3., 5., 3.5, 6.5, 4.5, 8.5}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.AVG);

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained
}
 

@Test
public void testSubSampleMaxActivate() throws Exception {
    INDArray containedExpectedOut =
                    Nd4j.create(new double[] {5., 7., 6., 8., 4., 7., 5., 9.}, new long[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, featureMapWidth, featureMapHeight},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4); // channels retained
}
 

@Test
public void testUpsampling() throws Exception {

    double[] outArray = new double[] {1., 1., 2., 2., 1., 1., 2., 2., 3., 3., 4., 4., 3., 3., 4., 4.};
    INDArray containedExpectedOut = Nd4j.create(outArray, new int[] {1, 1, 4, 4});
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer = getUpsamplingLayer();

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, outputWidth, outputHeight},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4);
}
 

@Test
public void testUpsampling1D() throws Exception {

    double[] outArray = new double[] {1., 1., 2., 2., 3., 3., 4., 4.};
    INDArray containedExpectedOut = Nd4j.create(outArray, new int[] {1, 1, 8});
    INDArray containedInput = getContainedData();
    INDArray input = getData();
    Layer layer =  getUpsampling1DLayer();

    INDArray containedOutput = layer.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(new long[] {nExamples, nChannelsIn, outputLength},
                    output.shape()));
    assertEquals(nChannelsIn, output.size(1), 1e-4);
}
 

@Test
public void testRepeatVector() {

    double[] arr = new double[] {1., 2., 3., 1., 2., 3., 1., 2., 3., 1., 2., 3.};
    INDArray expectedOut = Nd4j.create(arr, new long[] {1, 3, REPEAT}, 'f');
    INDArray input = Nd4j.create(new double[] {1., 2., 3.}, new long[] {1, 3});
    Layer layer = getRepeatVectorLayer();

    INDArray output = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());
    assertTrue(Arrays.equals(expectedOut.shape(), output.shape()));
    assertEquals(expectedOut, output);

    INDArray epsilon = Nd4j.ones(1,3,4);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    INDArray outEpsilon = out.getSecond();
    INDArray expectedEpsilon = Nd4j.create(new double[] {4., 4., 4.}, new long[] {1, 3});
    assertEquals(expectedEpsilon, outEpsilon);
}
 

@Test
public void testSpaceToDepthForward() throws Exception {
    INDArray containedInput = getContainedData();
    INDArray containedExpectedOut = getContainedOutput();
    Layer std = getSpaceToDepthLayer();
    INDArray containedOutput = std.activate(containedInput, false, LayerWorkspaceMgr.noWorkspaces());

    assertTrue(Arrays.equals(containedExpectedOut.shape(), containedOutput.shape()));
    assertEquals(containedExpectedOut, containedOutput);
}
 

@Test
public void testCNNInputSetupMNIST() throws Exception {
    INDArray input = getMnistData();
    Layer layer = getMNISTConfig();
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(input, layer.input());
    assertArrayEquals(input.shape(), layer.input().shape());
}
 

@Test
public void testActivateResultsContained() {
    Layer layer = getContainedConfig();
    INDArray input = getContainedData();
    INDArray expectedOutput = Nd4j.create(new float[] {0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f,
                    0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f,
                    0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f,
                    0.99966465f, 0.99966465f, 0.99966465f, 0.98201379f, 0.98201379f, 0.98201379f, 0.98201379f, 0.99966465f,
                    0.99966465f, 0.99966465f, 0.99966465f}, new int[] {1, 2, 4, 4});

    INDArray convActivations = layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    assertArrayEquals(expectedOutput.shape(), convActivations.shape());
    assertEquals(expectedOutput, convActivations);
}
 

@Test
public void testUpsampling1DBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 3., 2., 6., 7., 2., 5., 5.},
                            new int[] {1, 1, 8});

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 8., 9., 10.},
                    new int[] {1, 1, 4});

    INDArray input = getContainedData();

    Layer layer = getUpsampling1DLayer();
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    val depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, outputLength);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1));
}
 

@Test
public void testUpsampling2DBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.},
                            new int[] {1, 1, 4, 4});

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {4., 4., 4., 4.},
                    new int[] {1, 1, 2, 2});

    INDArray input = getContainedData();

    Layer layer = getUpsamplingLayer();
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    val depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, outputHeight, outputWidth);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1));
}
 

@Test
public void testCnnForwardPass() {
    int nOut = 10;
    Layer l = getLayer(nOut, 0.0, false, -1, -1);
    assertEquals(4 * nOut, l.numParams()); //Gamma, beta, global mean, global var
    int hw = 15;

    Nd4j.getRandom().setSeed(12345);
    INDArray randInput = Nd4j.rand(new int[]{100, nOut, hw, hw});
    INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());

    assertEquals(4, output.rank());

    INDArray mean = output.mean(0, 2, 3);
    INDArray stdev = output.std(false, 0, 2, 3);

    assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f);
    assertArrayEquals(Nd4j.ones(1, nOut).data().asFloat(), stdev.data().asFloat(), 1e-6f);

    //If we fix gamma/beta: expect different mean and variance...
    double gamma = 2.0;
    double beta = 3.0;
    l = getLayer(nOut, 0.0, true, gamma, beta);
    assertEquals(2 * nOut, l.numParams()); //Should have only global mean/var parameters
    output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());
    mean = output.mean(0, 2, 3);
    stdev = output.std(false, 0, 2, 3);

    assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean);
    assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev);
}
 

@Test
public void testSubSampleLayerMaxBackprop() throws Exception {
    INDArray expectedContainedEpsilonInput =
                    Nd4j.create(new double[] {1., 1., 1., 1., 1., 1., 1., 1.}, new int[] {1, 2, 2, 2}).castTo(Nd4j.defaultFloatingPointType());

    INDArray expectedContainedEpsilonResult = Nd4j.create(new double[] {0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1.,
                    0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0.},
                    new int[] {1, 2, 4, 4}).castTo(Nd4j.defaultFloatingPointType());

    INDArray input = getContainedData();

    Layer layer = getSubsamplingLayer(SubsamplingLayer.PoolingType.MAX);
    layer.activate(input, false, LayerWorkspaceMgr.noWorkspaces());

    Pair<Gradient, INDArray> containedOutput = layer.backpropGradient(expectedContainedEpsilonInput, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expectedContainedEpsilonResult, containedOutput.getSecond());
    assertEquals(null, containedOutput.getFirst().getGradientFor("W"));
    assertEquals(expectedContainedEpsilonResult.shape().length, containedOutput.getSecond().shape().length);

    INDArray input2 = getData();
    layer.activate(input2, false, LayerWorkspaceMgr.noWorkspaces());
    long depth = input2.size(1);

    epsilon = Nd4j.ones(5, depth, featureMapHeight, featureMapWidth);

    Pair<Gradient, INDArray> out = layer.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(input.shape().length, out.getSecond().shape().length);
    assertEquals(depth, out.getSecond().size(1)); // channels retained
}
 

@Test
    public void testDnnForwardPass() {
        int nOut = 10;
        Layer l = getLayer(nOut, 0.0, false, -1, -1);
        assertEquals(4 * nOut, l.numParams()); //Gamma, beta, global mean, global var

        INDArray randInput = Nd4j.rand(100, nOut);
        INDArray output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());

        INDArray mean = output.mean(0);
        INDArray stdev = output.std(false, 0);

//        System.out.println(Arrays.toString(mean.data().asFloat()));

        assertArrayEquals(new float[nOut], mean.data().asFloat(), 1e-6f);
        assertEquals(Nd4j.ones(nOut), stdev);

        //If we fix gamma/beta: expect different mean and variance...
        double gamma = 2.0;
        double beta = 3.0;
        l = getLayer(nOut, 0.0, true, gamma, beta);
        assertEquals(2 * nOut, l.numParams()); //Should have only global mean/var parameters
        output = l.activate(randInput, true, LayerWorkspaceMgr.noWorkspaces());
        mean = output.mean(0);
        stdev = output.std(false, 0);

        assertEquals(Nd4j.valueArrayOf(mean.shape(), beta), mean);
        assertEquals(Nd4j.valueArrayOf(stdev.shape(), gamma), stdev);
    }
 

@Test
public void testLastTimeStepVertex() {

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
            .addLayer("lastTS", new LastTimeStep(new SimpleRnn.Builder()
                    .nIn(5).nOut(6).dataFormat(rnnDataFormat).build()), "in")
            .setOutputs("lastTS")
            .build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    //First: test without input mask array
    Nd4j.getRandom().setSeed(12345);
    Layer l = graph.getLayer("lastTS");
    INDArray in;
    if (rnnDataFormat == RNNFormat.NCW){
        in = Nd4j.rand(3, 5, 6);
    }
    else{
        in = Nd4j.rand(3, 6, 5);
    }
    INDArray outUnderlying = ((LastTimeStepLayer)l).getUnderlying().activate(in, false, LayerWorkspaceMgr.noWorkspaces());
    INDArray expOut;
    if (rnnDataFormat == RNNFormat.NCW){
        expOut = outUnderlying.get(NDArrayIndex.all(), NDArrayIndex.all(), NDArrayIndex.point(5));
    }
    else{
        expOut = outUnderlying.get(NDArrayIndex.all(), NDArrayIndex.point(5), NDArrayIndex.all());
    }



    //Forward pass:
    INDArray outFwd = l.activate(in, false, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expOut, outFwd);

    //Second: test with input mask array
    INDArray inMask = Nd4j.zeros(3, 6);
    inMask.putRow(0, Nd4j.create(new double[]{1, 1, 1, 0, 0, 0}));
    inMask.putRow(1, Nd4j.create(new double[]{1, 1, 1, 1, 0, 0}));
    inMask.putRow(2, Nd4j.create(new double[]{1, 1, 1, 1, 1, 0}));
    graph.setLayerMaskArrays(new INDArray[]{inMask}, null);

    expOut = Nd4j.zeros(3, 6);
    if (rnnDataFormat == RNNFormat.NCW){
        expOut.putRow(0, outUnderlying.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(2)));
        expOut.putRow(1, outUnderlying.get(NDArrayIndex.point(1), NDArrayIndex.all(), NDArrayIndex.point(3)));
        expOut.putRow(2, outUnderlying.get(NDArrayIndex.point(2), NDArrayIndex.all(), NDArrayIndex.point(4)));
    }
    else{
        expOut.putRow(0, outUnderlying.get(NDArrayIndex.point(0), NDArrayIndex.point(2), NDArrayIndex.all()));
        expOut.putRow(1, outUnderlying.get(NDArrayIndex.point(1), NDArrayIndex.point(3), NDArrayIndex.all()));
        expOut.putRow(2, outUnderlying.get(NDArrayIndex.point(2), NDArrayIndex.point(4), NDArrayIndex.all()));
    }


    outFwd = l.activate(in, false, LayerWorkspaceMgr.noWorkspaces());
    assertEquals(expOut, outFwd);

    TestUtils.testModelSerialization(graph);
}
 

@Test
    public void testCnnForwardBackward() {
        double eps = 1e-5;
        int nIn = 4;
        int hw = 3;
        int minibatch = 2;
        Nd4j.getRandom().setSeed(12345);
        INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw});

        //TODO: other values for gamma/beta
        INDArray gamma = Nd4j.ones(1, nIn);
        INDArray beta = Nd4j.zeros(1, nIn);

        Layer l = getLayer(nIn, eps, false, -1, -1);

        INDArray mean = input.mean(0, 2, 3);
        INDArray var = input.var(false, 0, 2, 3);
        INDArray xHat = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));
        Nd4j.getExecutioner().exec(new BroadcastDivOp(xHat, Transforms.sqrt(var.add(eps), true), xHat, 1));

        INDArray outExpected = Nd4j.getExecutioner().exec(new BroadcastMulOp(xHat, gamma, xHat.dup(), 1));
        Nd4j.getExecutioner().exec(new BroadcastAddOp(outExpected, beta, outExpected, 1));

        INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces());

//        System.out.println(Arrays.toString(outExpected.data().asDouble()));
//        System.out.println(Arrays.toString(out.data().asDouble()));

        assertEquals(outExpected, out);

        //-------------------------------------------------------------
        //Check backprop
        INDArray epsilon = Nd4j.rand('c', new int[]{minibatch, nIn, hw, hw}); //dL/dy

        int effectiveMinibatch = minibatch * hw * hw;

        INDArray dldgammaExp = epsilon.mul(xHat).sum(0, 2, 3);
        dldgammaExp = dldgammaExp.reshape(1, dldgammaExp.length());
        INDArray dldbetaExp = epsilon.sum(0, 2, 3);
        dldbetaExp = dldbetaExp.reshape(1, dldbetaExp.length());

        INDArray dldxhat = Nd4j.getExecutioner().exec(new BroadcastMulOp(epsilon, gamma, epsilon.dup(), 1)); //epsilon.mulRowVector(gamma);

        INDArray inputSubMean = Nd4j.getExecutioner().exec(new BroadcastSubOp(input, mean, input.dup(), 1));

        INDArray dldvar = dldxhat.mul(inputSubMean).mul(-0.5);
        dldvar = Nd4j.getExecutioner().exec(
                new BroadcastMulOp(dldvar, Transforms.pow(var.add(eps), -3.0 / 2.0, true), dldvar.dup(), 1));
        dldvar = dldvar.sum(0, 2, 3);


        INDArray dldmu = Nd4j
                .getExecutioner().exec(new BroadcastMulOp(dldxhat,
                        Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1))
                .neg().sum(0, 2, 3);
        dldmu = dldmu.add(dldvar.mul(inputSubMean.mul(-2.0).sum(0, 2, 3).div(effectiveMinibatch)));

        INDArray dldinExp = Nd4j.getExecutioner().exec(
                new BroadcastMulOp(dldxhat, Transforms.pow(var.add(eps), -1.0 / 2.0, true), dldxhat.dup(), 1));
        dldinExp = dldinExp.add(Nd4j.getExecutioner().exec(
                new BroadcastMulOp(inputSubMean.mul(2.0 / effectiveMinibatch), dldvar, inputSubMean.dup(), 1)));
        dldinExp = Nd4j.getExecutioner().exec(
                new BroadcastAddOp(dldinExp, dldmu.mul(1.0 / effectiveMinibatch), dldinExp.dup(), 1));

        Pair<Gradient, INDArray> p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());

        INDArray dldgamma = p.getFirst().getGradientFor("gamma");
        INDArray dldbeta = p.getFirst().getGradientFor("beta");

        assertEquals(dldgammaExp, dldgamma);
        assertEquals(dldbetaExp, dldbeta);

        //        System.out.println("EPSILONS");
        //        System.out.println(Arrays.toString(dldinExp.data().asDouble()));
        //        System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
        assertEquals(dldinExp, p.getSecond());
    }
 

@Test
public void test2dVs4d() {
    //Idea: 2d and 4d should be the same...
    Nd4j.getRandom().setSeed(12345);

    int m = 2;
    int h = 3;
    int w = 3;
    int nOut = 2;

    INDArray in = Nd4j.rand('c', m * h * w, nOut);

    INDArray in4 = in.dup();
    in4 = Shape.newShapeNoCopy(in4, new int[]{m, h, w, nOut}, false);
    assertNotNull(in4);
    in4 = in4.permute(0, 3, 1, 2).dup();
    INDArray arr = Nd4j.rand(1, m * h * w * nOut).reshape('f', h, w, m, nOut).permute(2, 3, 1, 0);
    in4 = arr.assign(in4);

    Layer l1 = getLayer(nOut);
    Layer l2 = getLayer(nOut);

    INDArray out2d = l1.activate(in.dup(), true, LayerWorkspaceMgr.noWorkspaces());
    INDArray out4d = l2.activate(in4.dup(), true, LayerWorkspaceMgr.noWorkspaces());

    INDArray out4dAs2 = out4d.permute(0, 2, 3, 1).dup('c');
    out4dAs2 = Shape.newShapeNoCopy(out4dAs2, new int[]{m * h * w, nOut}, false);

    assertEquals(out2d, out4dAs2);

    //Test backprop:
    INDArray epsilons2d = Nd4j.rand('c', m * h * w, nOut);
    INDArray epsilons4d = epsilons2d.dup();
    epsilons4d = Shape.newShapeNoCopy(epsilons4d, new int[]{m, h, w, nOut}, false);
    assertNotNull(epsilons4d);
    epsilons4d = epsilons4d.permute(0, 3, 1, 2).dup();

    Pair<Gradient, INDArray> b2d = l1.backpropGradient(epsilons2d, LayerWorkspaceMgr.noWorkspaces());
    Pair<Gradient, INDArray> b4d = l2.backpropGradient(epsilons4d, LayerWorkspaceMgr.noWorkspaces());

    INDArray e4dAs2d = b4d.getSecond().permute(0, 2, 3, 1).dup('c');
    e4dAs2d = Shape.newShapeNoCopy(e4dAs2d, new int[]{m * h * w, nOut}, false);

    assertEquals(b2d.getSecond(), e4dAs2d);
}
 

@Test
    public void testDnnForwardBackward() {
        double eps = 1e-5;
        int nIn = 4;
        int minibatch = 2;
        Nd4j.getRandom().setSeed(12345);
        INDArray input = Nd4j.rand('c', new int[]{minibatch, nIn});

        //TODO: other values for gamma/beta
        INDArray gamma = Nd4j.ones(1, nIn);
        INDArray beta = Nd4j.zeros(1, nIn);

        Layer l = getLayer(nIn, eps, false, -1, -1);

        INDArray mean = input.mean(0);
        INDArray var = input.var(false, 0);
        INDArray xHat = input.subRowVector(mean).divRowVector(Transforms.sqrt(var.add(eps), true));
        INDArray outExpected = xHat.mulRowVector(gamma).addRowVector(beta);

        INDArray out = l.activate(input, true, LayerWorkspaceMgr.noWorkspaces());

//        System.out.println(Arrays.toString(outExpected.data().asDouble()));
//        System.out.println(Arrays.toString(out.data().asDouble()));

        assertEquals(outExpected, out);

        //-------------------------------------------------------------
        //Check backprop
        INDArray epsilon = Nd4j.rand(minibatch, nIn); //dL/dy

        INDArray dldgammaExp = epsilon.mul(xHat).sum(true, 0);
        INDArray dldbetaExp = epsilon.sum(true, 0);

        INDArray dldxhat = epsilon.mulRowVector(gamma);
        INDArray dldvar = dldxhat.mul(input.subRowVector(mean)).mul(-0.5)
                .mulRowVector(Transforms.pow(var.add(eps), -3.0 / 2.0, true)).sum(0);
        INDArray dldmu = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true)).neg().sum(0)
                .add(dldvar.mul(input.subRowVector(mean).mul(-2.0).sum(0).div(minibatch)));
        INDArray dldinExp = dldxhat.mulRowVector(Transforms.pow(var.add(eps), -1.0 / 2.0, true))
                .add(input.subRowVector(mean).mul(2.0 / minibatch).mulRowVector(dldvar))
                .addRowVector(dldmu.mul(1.0 / minibatch));

        Pair<Gradient, INDArray> p = l.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());

        INDArray dldgamma = p.getFirst().getGradientFor("gamma");
        INDArray dldbeta = p.getFirst().getGradientFor("beta");

        assertEquals(dldgammaExp, dldgamma);
        assertEquals(dldbetaExp, dldbeta);

//        System.out.println("EPSILONS");
//        System.out.println(Arrays.toString(dldinExp.data().asDouble()));
//        System.out.println(Arrays.toString(p.getSecond().dup().data().asDouble()));
        assertEquals(dldinExp, p.getSecond());
    }