Java Code Examples for org.deeplearning4j.nn.multilayer.MultiLayerNetwork#getParam()

The following examples show how to use org.deeplearning4j.nn.multilayer.MultiLayerNetwork#getParam() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestSimpleRnn.java From deeplearning4j with Apache License 2.0

6 votes

@Test
public void testBiasInit(){
    Nd4j.getRandom().setSeed(12345);
    int nIn = 5;
    int layerSize = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new NoOp())
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.TANH)
            .list()
            .layer(new SimpleRnn.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat)
                    .biasInit(100)
                    .build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray bArr = net.getParam("0_b");
    assertEquals(Nd4j.valueArrayOf(new long[]{1,layerSize}, 100.0f), bArr);
}

Example 2

Source File: Word2VecTestsSmall.java From deeplearning4j with Apache License 2.0

4 votes

@Test(timeout = 300000)
    public void testW2VEmbeddingLayerInit() throws Exception {
        Nd4j.setDefaultDataTypes(DataType.FLOAT, DataType.FLOAT);

        val inputFile = Resources.asFile("big/raw_sentences.txt");
        val iter = ParagraphVectorsTest.getIterator(isIntegrationTests(), inputFile);
//        val iter = new BasicLineIterator(inputFile);
        val t = new DefaultTokenizerFactory();
        t.setTokenPreProcessor(new CommonPreprocessor());

        Word2Vec vec = new Word2Vec.Builder()
                .minWordFrequency(1)
                .epochs(1)
                .layerSize(300)
                .limitVocabularySize(1) // Limit the vocab size to 2 words
                .windowSize(5)
                .allowParallelTokenization(true)
                .batchSize(512)
                .learningRate(0.025)
                .minLearningRate(0.0001)
                .negativeSample(0.0)
                .sampling(0.0)
                .useAdaGrad(false)
                .useHierarchicSoftmax(true)
                .iterations(1)
                .useUnknown(true) // Using UNK with limited vocab size causes the issue
                .seed(42)
                .iterate(iter)
                .workers(4)
                .tokenizerFactory(t).build();

        vec.fit();

        INDArray w = vec.lookupTable().getWeights();
        System.out.println(w);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345).list()
                .layer(new EmbeddingLayer.Builder().weightInit(vec).build())
                .layer(new DenseLayer.Builder().activation(Activation.TANH).nIn(w.size(1)).nOut(3).build())
                .layer(new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(3)
                        .nOut(4).build())
                .build();

        final MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray w0 = net.getParam("0_W");
        assertEquals(w, w0);

        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ModelSerializer.writeModel(net, baos, true);
        byte[] bytes = baos.toByteArray();

        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
        final MultiLayerNetwork restored = ModelSerializer.restoreMultiLayerNetwork(bais, true);

        assertEquals(net.getLayerWiseConfigurations(), restored.getLayerWiseConfigurations());
        assertTrue(net.params().equalsWithEps(restored.params(), 2e-3));
    }

Example 3

Source File: TestRecurrentWeightInit.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void testRWInit() {

    for (boolean rwInit : new boolean[]{false, true}) {
        for (int i = 0; i < 3; i++) {

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder()
                    .weightInit(new UniformDistribution(0, 1))
                    .list();

            if(rwInit) {
                switch (i) {
                    case 0:
                        b.layer(new LSTM.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3))
                                .build());
                        break;
                    case 1:
                        b.layer(new GravesLSTM.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3))
                                .build());
                        break;
                    case 2:
                        b.layer(new SimpleRnn.Builder().nIn(10).nOut(10)
                                .weightInitRecurrent(new UniformDistribution(2, 3)).build());
                        break;
                    default:
                        throw new RuntimeException();
                }
            } else {
                switch (i) {
                    case 0:
                        b.layer(new LSTM.Builder().nIn(10).nOut(10).build());
                        break;
                    case 1:
                        b.layer(new GravesLSTM.Builder().nIn(10).nOut(10).build());
                        break;
                    case 2:
                        b.layer(new SimpleRnn.Builder().nIn(10).nOut(10).build());
                        break;
                    default:
                        throw new RuntimeException();
                }
            }

            MultiLayerNetwork net = new MultiLayerNetwork(b.build());
            net.init();

            INDArray rw = net.getParam("0_RW");
            double min = rw.minNumber().doubleValue();
            double max = rw.maxNumber().doubleValue();
            if(rwInit){
                assertTrue(String.valueOf(min), min >= 2.0);
                assertTrue(String.valueOf(max), max <= 3.0);
            } else {
                assertTrue(String.valueOf(min), min >= 0.0);
                assertTrue(String.valueOf(max), max <= 1.0);
            }
        }
    }
}

Example 4

Source File: TestSimpleRnn.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void testSimpleRnn(){
    Nd4j.getRandom().setSeed(12345);

    int m = 3;
    int nIn = 5;
    int layerSize = 6;
    int tsLength = 7;
    INDArray in;
    if (rnnDataFormat == RNNFormat.NCW){
        in = Nd4j.rand(DataType.FLOAT, m, nIn, tsLength);
    }
    else{
        in = Nd4j.rand(DataType.FLOAT, m, tsLength, nIn);
    }


    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new NoOp())
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.TANH)
            .list()
            .layer(new SimpleRnn.Builder().nIn(nIn).nOut(layerSize).dataFormat(rnnDataFormat).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray out = net.output(in);

    INDArray w = net.getParam("0_W");
    INDArray rw = net.getParam("0_RW");
    INDArray b = net.getParam("0_b");

    INDArray outLast = null;
    for( int i=0; i<tsLength; i++ ){
        INDArray inCurrent;
        if (rnnDataFormat == RNNFormat.NCW){
            inCurrent = in.get(all(), all(), point(i));
        }
        else{
            inCurrent = in.get(all(), point(i), all());
        }

        INDArray outExpCurrent = inCurrent.mmul(w);
        if(outLast != null){
            outExpCurrent.addi(outLast.mmul(rw));
        }

        outExpCurrent.addiRowVector(b);

        Transforms.tanh(outExpCurrent, false);

        INDArray outActCurrent;
        if (rnnDataFormat == RNNFormat.NCW){
            outActCurrent = out.get(all(), all(), point(i));
        }
        else{
            outActCurrent = out.get(all(), point(i), all());
        }
        assertEquals(String.valueOf(i), outExpCurrent, outActCurrent);

        outLast = outExpCurrent;
    }


    TestUtils.testModelSerialization(net);
}

Example 5

Source File: BatchNormalizationTest.java From deeplearning4j with Apache License 2.0

4 votes

@Test
public void checkMeanVarianceEstimateCNNCompareModes() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    //Check that the internal global mean/variance estimate is approximately correct

    //First, Mnist data as 2d input (NOT taking into account convolution property)
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(Updater.RMSPROP).seed(12345).list()
            .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(false).build())
            .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).nOut(10).build())
            .setInputType(InputType.convolutional(5, 5, 3)).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
            .updater(Updater.RMSPROP).seed(12345).list()
            .layer(0, new BatchNormalization.Builder().nIn(3).nOut(3).eps(1e-5).decay(0.95).useLogStd(true).build())
            .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MSE).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).nOut(10).build())
            .setInputType(InputType.convolutional(5, 5, 3)).build();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    int minibatch = 32;
    for (int i = 0; i < 10; i++) {
        DataSet ds = new DataSet(Nd4j.rand(new int[]{minibatch, 3, 5, 5}), Nd4j.rand(minibatch, 10));
        net.fit(ds);
        net2.fit(ds);

        INDArray globalVar = net.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_VAR);

        INDArray log10std = net2.getParam("0_" + BatchNormalizationParamInitializer.GLOBAL_LOG_STD);
        INDArray globalVar2 = Nd4j.valueArrayOf(log10std.shape(), 10.0).castTo(log10std.dataType());
        Transforms.pow(globalVar2, log10std, false);    // stdev = 10^(log10(stdev))
        globalVar2.muli(globalVar2);

        assertEquals(globalVar, globalVar2);
    }
}

Example 6