org.deeplearning4j.nn.conf.NeuralNetConfiguration Java Examples

The following examples show how to use org.deeplearning4j.nn.conf.NeuralNetConfiguration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CustomPreprocessorTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
    public void testCustomPreprocessor() {
        //Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works...
        MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().list()
                                        .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
                                        .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10)
                                                .activation(Activation.SOFTMAX).nOut(10).build())
                                        .inputPreProcessor(0, new MyCustomPreprocessor())
                                        .build();

        String json = conf.toJson();
        String yaml = conf.toYaml();

//        System.out.println(json);

        MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json);
        assertEquals(conf, confFromJson);

        MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml);
        assertEquals(conf, confFromYaml);

        assertTrue(confFromJson.getInputPreProcess(0) instanceof MyCustomPreprocessor);

    }
 
Example #2
Source File: ConvolutionLayerSetupTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiChannel() throws Exception {
    INDArray in = Nd4j.rand(new int[] {10, 3, 28, 28});
    INDArray labels = Nd4j.rand(10, 2);
    DataSet next = new DataSet(in, labels);

    NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLFW();
    builder.setInputType(InputType.convolutional(28, 28, 3));
    MultiLayerConfiguration conf = builder.build();
    ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(2).getLayer();
    assertEquals(6, layer2.getNIn());

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.fit(next);
}
 
Example #3
Source File: TestListeners.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testListenersGraph() {
    TestListener.clearCounts();

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder()
                    .addInputs("in").addLayer("0",
                                    new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10)
                                                    .activation(Activation.TANH).build(),
                                    "in")
                    .setOutputs("0").build();

    ComputationGraph model = new ComputationGraph(conf);
    model.init();

    testListenersForModel(model, Collections.singletonList(new TestListener()));
}
 
Example #4
Source File: ModelSerializerTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteMLNModel() throws Exception {
    int nIn = 5;
    int nOut = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01)
                    .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build())
                    .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    File tempFile = tempDir.newFile();

    ModelSerializer.writeModel(net, tempFile, true);

    MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(tempFile);

    assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson());
    assertEquals(net.params(), network.params());
    assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray());
}
 
Example #5
Source File: FrozenLayerWithBackprop.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf,
                                                   Collection<TrainingListener> trainingListeners, int layerIndex, INDArray layerParamsView,
                                                   boolean initializeParams, DataType networkDataType) {

    //Need to be able to instantiate a layer, from a config - for JSON -> net type situations
    org.deeplearning4j.nn.api.Layer underlying = getUnderlying().instantiate(getInnerConf(conf), trainingListeners,
                    layerIndex, layerParamsView, initializeParams, networkDataType);

    NeuralNetConfiguration nncUnderlying = underlying.conf();

    if (nncUnderlying.variables() != null) {
        List<String> vars = nncUnderlying.variables(true);
        nncUnderlying.clearVariables();
        conf.clearVariables();
        for (String s : vars) {
            conf.variables(false).add(s);
            nncUnderlying.variables(false).add(s);
        }
    }

    return new org.deeplearning4j.nn.layers.FrozenLayerWithBackprop(underlying);
}
 
Example #6
Source File: SameDiffCustomLayerTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testInputValidationSameDiffVertex(){
    final ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().graphBuilder()
            .addVertex("a", new ValidatingSameDiffVertex(), "input")
            .addLayer("output", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build(), "a")
            .addInputs("input")
            .setInputTypes(InputType.feedForward(2))
            .setOutputs("output")
            .build();

    final ComputationGraph net = new ComputationGraph(config);
    net.init();

    final INDArray goodInput = Nd4j.rand(1, 2);
    final INDArray badInput = Nd4j.rand(2, 2);

    net.fit(new INDArray[]{goodInput}, new INDArray[]{goodInput});

    exceptionRule.expect(IllegalArgumentException.class);
    exceptionRule.expectMessage("Expected Message");
    net.fit(new INDArray[]{badInput}, new INDArray[]{badInput});
}
 
Example #7
Source File: SameDiffVertex.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public void applyGlobalConfig(NeuralNetConfiguration.Builder b) {
    if(regularization == null || regularization.isEmpty()){
        regularization = b.getRegularization();
    }
    if(regularizationBias == null || regularizationBias.isEmpty()){
        regularizationBias = b.getRegularizationBias();
    }
    if (updater == null) {
        updater = b.getIUpdater();
    }
    if (biasUpdater == null) {
        biasUpdater = b.getBiasUpdater();
    }
    if (gradientNormalization == null) {
        gradientNormalization = b.getGradientNormalization();
    }
    if (Double.isNaN(gradientNormalizationThreshold)) {
        gradientNormalizationThreshold = b.getGradientNormalizationThreshold();
    }

    applyGlobalConfigToLayer(b);
}
 
Example #8
Source File: TestUpdaters.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetGetUpdater2() {
    //Same as above test, except that we are doing setUpdater on a new network
    Nd4j.getRandom().setSeed(12345L);
    double lr = 0.03;
    int nIn = 4;
    int nOut = 8;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(lr,0.6)).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(5)
                                    .updater(org.deeplearning4j.nn.conf.Updater.SGD).build())
                    .layer(1, new DenseLayer.Builder().nIn(5).nOut(6)
                                    .updater(new NoOp()).build())
                    .layer(2, new DenseLayer.Builder().nIn(6).nOut(7)
                                    .updater(org.deeplearning4j.nn.conf.Updater.ADAGRAD).build())
                    .layer(3, new OutputLayer.Builder().nIn(7).nOut(nOut).activation(Activation.SOFTMAX)
                                    .updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Updater newUpdater = UpdaterCreator.getUpdater(net);
    net.setUpdater(newUpdater);
    assertTrue(newUpdater == net.getUpdater()); //Should be identical object
}
 
Example #9
Source File: SameDiffCustomLayerTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testInputValidationSameDiffLayer(){
    final MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().list()
            .layer(new ValidatingSameDiffLayer())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build())
            .setInputType(InputType.feedForward(2))
            .build();

    final MultiLayerNetwork net = new MultiLayerNetwork(config);
    net.init();

    final INDArray goodInput = Nd4j.rand(1, 2);
    final INDArray badInput = Nd4j.rand(2, 2);

    net.fit(goodInput, goodInput);

   exceptionRule.expect(IllegalArgumentException.class);
   exceptionRule.expectMessage("Expected Message");
   net.fit(badInput, badInput);
}
 
Example #10
Source File: WeightInitIdentityTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * Test identity mapping for 1d convolution
 */
@Test
public void testIdConv1D() {
    final INDArray input = Nd4j.randn(DataType.FLOAT, 1,5,7);
    final String inputName = "input";
    final String conv = "conv";
    final String output = "output";
    final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .setInputTypes(InputType.inferInputType(input))
            .addInputs(inputName)
            .setOutputs(output)
            .layer(conv, new Convolution1DLayer.Builder(7)
                    .convolutionMode(ConvolutionMode.Same)
                    .nOut(input.size(1))
                    .weightInit(new WeightInitIdentity())
                    .activation(new ActivationIdentity())
                    .build(), inputName)
            .layer(output, new RnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv)
            .build());
    graph.init();

    assertEquals("Mapping was not identity!", input, graph.outputSingle(input).reshape(input.shape()));
}
 
Example #11
Source File: LayerConfigTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testUpdaterAdamParamsLayerwiseOverride() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1.0, 0.5, 0.5, 1e-8))
            .list()
            .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
                    .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Adam(1.0, 0.6, 0.7, 1e-8)).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.6, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0);
    assertEquals(0.7, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0);
}
 
Example #12
Source File: SpaceToDepthLayer.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf,
                                                   Collection<TrainingListener> trainingListeners, int layerIndex, INDArray layerParamsView,
                                                   boolean initializeParams, DataType networkDataType) {
    org.deeplearning4j.nn.layers.convolution.SpaceToDepth ret =
                    new org.deeplearning4j.nn.layers.convolution.SpaceToDepth(conf, networkDataType);
    ret.setListeners(trainingListeners);
    ret.setIndex(layerIndex);
    ret.setParamsViewArray(layerParamsView);
    Map<String, INDArray> paramTable = initializer().init(conf, layerParamsView, initializeParams);
    ret.setParamTable(paramTable);
    ret.setConf(conf);
    return ret;
}
 
Example #13
Source File: GravesLSTM.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public Layer instantiate(NeuralNetConfiguration conf, Collection<TrainingListener> trainingListeners,
                         int layerIndex, INDArray layerParamsView, boolean initializeParams, DataType networkDataType) {
    LayerValidation.assertNInNOutSet("GravesLSTM", getLayerName(), layerIndex, getNIn(), getNOut());
    org.deeplearning4j.nn.layers.recurrent.GravesLSTM ret =
                    new org.deeplearning4j.nn.layers.recurrent.GravesLSTM(conf, networkDataType);
    ret.setListeners(trainingListeners);
    ret.setIndex(layerIndex);
    ret.setParamsViewArray(layerParamsView);
    Map<String, INDArray> paramTable = initializer().init(conf, layerParamsView, initializeParams);
    ret.setParamTable(paramTable);
    ret.setConf(conf);
    return ret;
}
 
Example #14
Source File: CenterLossOutputLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public ComputationGraph getCNNMnistConfig() {

        int nChannels = 1; // Number of input channels
        int outputNum = 10; // The number of possible outcomes

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) // Training iterations as above
                        .l2(0.0005).weightInit(WeightInit.XAVIER)
                        .updater(new Nesterovs(0.01, 0.9))
                        .graphBuilder().addInputs("input")
                        .setInputTypes(InputType.convolutionalFlat(28, 28, 1))
                        .addLayer("0", new ConvolutionLayer.Builder(5, 5)
                                        //nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied
                                        .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(),
                                        "input")
                        .addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "0")
                        .addLayer("2", new ConvolutionLayer.Builder(5, 5)
                                        //Note that nIn need not be specified in later layers
                                        .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1")
                        .addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "2")
                        .addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3")
                        .addLayer("output",
                                        new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder(
                                                        LossFunction.MCXENT).nOut(outputNum)
                                                                        .activation(Activation.SOFTMAX).build(),
                                        "4")
                        .setOutputs("output").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        return graph;
    }
 
Example #15
Source File: MaskLayer.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf,
                                                   Collection<TrainingListener> trainingListeners, int layerIndex, INDArray layerParamsView,
                                                   boolean initializeParams, DataType networkDataType) {
    org.deeplearning4j.nn.layers.util.MaskLayer ret = new org.deeplearning4j.nn.layers.util.MaskLayer(conf, networkDataType);
    ret.setIndex(layerIndex);
    ret.setParamsViewArray(layerParamsView);
    Map<String, INDArray> paramTable = initializer().init(conf, layerParamsView, initializeParams);
    ret.setParamTable(paramTable);
    ret.setConf(conf);
    return ret;
}
 
Example #16
Source File: TestSetGetParameters.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testSetParameters() {
    //Set up a MLN, then do set(get) on parameters. Results should be identical compared to before doing this.
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new DenseLayer.Builder().nIn(9).nOut(10)
                                    .dist(new NormalDistribution(0, 1)).build())
                    .layer(1, new DenseLayer.Builder().nIn(10).nOut(11)
                                    .dist(new NormalDistribution(0, 1)).build())
                    .layer(2, new AutoEncoder.Builder().corruptionLevel(0.5).nIn(11).nOut(12)
                                    .dist(new NormalDistribution(0, 1)).build())
                    .layer(3, new OutputLayer.Builder(LossFunction.MSE).nIn(12).nOut(12)
                                    .dist(new NormalDistribution(0, 1)).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    INDArray initParams = net.params().dup();
    Map<String, INDArray> initParams2 = net.paramTable();

    net.setParams(net.params());

    INDArray initParamsAfter = net.params();
    Map<String, INDArray> initParams2After = net.paramTable();

    for (String s : initParams2.keySet()) {
        assertTrue("Params differ: " + s, initParams2.get(s).equals(initParams2After.get(s)));
    }

    assertEquals(initParams, initParamsAfter);

    //Now, try the other way: get(set(random))
    INDArray randomParams = Nd4j.rand(initParams.dataType(), initParams.shape());
    net.setParams(randomParams.dup());

    assertEquals(net.params(), randomParams);
}
 
Example #17
Source File: LayerConfigValidationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testWeightInitDistNotSet() {
    // Warning thrown only since global dist can be set with a different weight init locally
    MultiLayerConfiguration conf =
            new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).dist(new GaussianDistribution(1e-3, 2))
                    .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
                    .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
}
 
Example #18
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void checkSerialization() throws Exception {
    //Serialize the batch norm network (after training), and make sure we get same activations out as before
    // i.e., make sure state is properly stored

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .list()
            .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).build())
            .layer(1, new BatchNormalization.Builder().build())
            .layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build())
            .layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build())
            .layer(4, new BatchNormalization.Builder().build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator iter = new MnistDataSetIterator(16, true, 12345);
    for (int i = 0; i < 20; i++) {
        net.fit(iter.next());
    }

    INDArray in = iter.next().getFeatures();

    INDArray out = net.output(in, false);
    INDArray out2 = net.output(in, false);

    assertEquals(out, out2);

    MultiLayerNetwork net2 = TestUtils.testModelSerialization(net);

    INDArray outDeser = net2.output(in, false);

    assertEquals(out, outDeser);
}
 
Example #19
Source File: ModelTupleStreamIntegrationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
private static Model buildModel() throws Exception {

    final int numInputs = 3;
    final int numOutputs = 2;

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 };
    // positive weight for first output, negative weight for second output, no biases
    assertEquals((numInputs+1)*numOutputs, floats.length);

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example #20
Source File: RandomTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testRngInitMLN() {
    Nd4j.getRandom().setSeed(12345);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
                    .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2,
                                    new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                    .activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
                    .build();

    String json = conf.toJson();

    MultiLayerNetwork net1 = new MultiLayerNetwork(conf);
    net1.init();

    MultiLayerNetwork net2 = new MultiLayerNetwork(conf);
    net2.init();

    assertEquals(net1.params(), net2.params());

    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json);

    Nd4j.getRandom().setSeed(987654321);
    MultiLayerNetwork net3 = new MultiLayerNetwork(fromJson);
    net3.init();

    assertEquals(net1.params(), net3.params());
}
 
Example #21
Source File: ModelSerializerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidLoading2() throws Exception {
    int nIn = 5;
    int nOut = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01)
            .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list()
            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build())
            .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder()
                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    File tempFile = tempDir.newFile("testInvalidLoading2.bin");

    ModelSerializer.writeModel(net, tempFile, true);

    try {
        ModelSerializer.restoreComputationGraph(tempFile);
        fail();
    } catch (Exception e){
        String msg = e.getMessage();
        assertTrue(msg, msg.contains("JSON") && msg.contains("restoreMultiLayerNetwork"));
    }
}
 
Example #22
Source File: EvaluationToolsTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testRocMultiToHtml() throws Exception {
        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                        .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1,
                                        new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                        .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        NormalizerStandardize ns = new NormalizerStandardize();
        DataSet ds = iter.next();
        ns.fit(ds);
        ns.transform(ds);

        for (int i = 0; i < 30; i++) {
            net.fit(ds);
        }

        for (int numSteps : new int[] {20, 0}) {
            ROCMultiClass roc = new ROCMultiClass(numSteps);
            iter.reset();

            INDArray f = ds.getFeatures();
            INDArray l = ds.getLabels();
            INDArray out = net.output(f);
            roc.eval(l, out);


            String str = EvaluationTools.rocChartToHtml(roc, Arrays.asList("setosa", "versicolor", "virginica"));
//            System.out.println(str);
        }
    }
 
Example #23
Source File: LSTMParamInitializer.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    org.deeplearning4j.nn.conf.layers.LSTM layerConf = (org.deeplearning4j.nn.conf.layers.LSTM) conf.getLayer();

    val nL = layerConf.getNOut(); //i.e., n neurons in this layer
    val nLast = layerConf.getNIn(); //i.e., n neurons in previous layer

    val length = numParams(conf);
    if (gradientView.length() != length)
        throw new IllegalStateException(
                        "Expected gradient view of length " + length + ", got length " + gradientView.length());

    val nParamsIn = nLast * (4 * nL);
    val nParamsRecurrent = nL * (4 * nL);
    val nBias = 4 * nL;
    INDArray inputWeightGradView = gradientView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nParamsIn))
                    .reshape('f', nLast, 4 * nL);
    INDArray recurrentWeightGradView = gradientView
                    .get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nParamsIn, nParamsIn + nParamsRecurrent))
                    .reshape('f', nL, 4 * nL);
    INDArray biasGradView = gradientView.get(NDArrayIndex.interval(0,0,true),
                    NDArrayIndex.interval(nParamsIn + nParamsRecurrent, nParamsIn + nParamsRecurrent + nBias)); //already a row vector

    Map<String, INDArray> out = new LinkedHashMap<>();
    out.put(INPUT_WEIGHT_KEY, inputWeightGradView);
    out.put(RECURRENT_WEIGHT_KEY, recurrentWeightGradView);
    out.put(BIAS_KEY, biasGradView);

    return out;
}
 
Example #24
Source File: TestSparkMultiLayerParameterAveraging.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
@Ignore    //Ignored 2019/04/09 - low priority: https://github.com/deeplearning4j/deeplearning4j/issues/6656
public void testVaePretrainSimpleCG() {
    //Simple sanity check on pretraining
    int nIn = 8;

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp())
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12)
                                    .decoderLayerSizes(13).reconstructionDistribution(
                                                    new GaussianReconstructionDistribution(Activation.IDENTITY))
                                    .build(), "in")
                    .setOutputs("0").build();

    //Do training on Spark with one executor, for 3 separate minibatches
    int rddDataSetNumExamples = 10;
    int totalAveragings = 5;
    int averagingFrequency = 3;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples)
                    .averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples)
                    .saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);
    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), tm);

    List<DataSet> trainData = new ArrayList<>();
    int nDataSets = numExecutors() * totalAveragings * averagingFrequency;
    for (int i = 0; i < nDataSets; i++) {
        trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null));
    }

    JavaRDD<DataSet> data = sc.parallelize(trainData);

    sparkNet.fit(data);
}
 
Example #25
Source File: GradientCheckTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testEmbeddingLayerPreluSimple() {
        Random r = new Random(12345);
        int nExamples = 5;
        INDArray input = Nd4j.zeros(nExamples, 1);
        INDArray labels = Nd4j.zeros(nExamples, 3);
        for (int i = 0; i < nExamples; i++) {
            input.putScalar(i, r.nextInt(4));
            labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
        }

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
                .dataType(DataType.DOUBLE)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
                .list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                .updater(new NoOp()).build())
                .layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
                .layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
                        .weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
                        .updater(new NoOp()).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        if (PRINT_RESULTS) {
            System.out.println("testEmbeddingLayerSimple");
//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        String msg = "testEmbeddingLayerSimple";
        assertTrue(msg, gradOK);
    }
 
Example #26
Source File: ElementWiseParamInitializer.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the parameters
 *
 * @param conf             the configuration
 * @param paramsView       a view of the full network (backprop) parameters
 * @param initializeParams if true: initialize the parameters according to the configuration. If false: don't modify the
 *                         values in the paramsView array (but do select out the appropriate subset, reshape etc as required)
 * @return Map of parameters keyed by type (view of the 'paramsView' array)
 */
@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    if (!(conf.getLayer() instanceof org.deeplearning4j.nn.conf.layers.FeedForwardLayer))
        throw new IllegalArgumentException("unsupported layer type: " + conf.getLayer().getClass().getName());

    Map<String, INDArray> params = Collections.synchronizedMap(new LinkedHashMap<String, INDArray>());

    val length = numParams(conf);
    if (paramsView.length() != length)
        throw new IllegalStateException(
                "Expected params view of length " + length + ", got length " + paramsView.length());

    org.deeplearning4j.nn.conf.layers.FeedForwardLayer layerConf =
            (org.deeplearning4j.nn.conf.layers.FeedForwardLayer) conf.getLayer();
    val nIn = layerConf.getNIn();

    val nWeightParams = nIn ;
    INDArray weightView = paramsView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nWeightParams));
    INDArray biasView = paramsView.get(NDArrayIndex.interval(0,0,true),
            NDArrayIndex.interval(nWeightParams, nWeightParams + nIn));


    params.put(WEIGHT_KEY, createWeightMatrix(conf, weightView, initializeParams));
    params.put(BIAS_KEY, createBias(conf, biasView, initializeParams));
    conf.addVariable(WEIGHT_KEY);
    conf.addVariable(BIAS_KEY);

    return params;
}
 
Example #27
Source File: TestEarlyStoppingCompGraph.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                    new ScoreImprovementEpochTerminationCondition(5))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                    .scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 
Example #28
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testDBNBNMultiLayer() throws Exception {
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    // Run with separate activation layer
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .list()
            .layer(0, new DenseLayer.Builder().nIn(28 * 28).nOut(10).weightInit(WeightInit.XAVIER)
                    .activation(Activation.RELU).build())
            .layer(1, new BatchNormalization.Builder().nOut(10).build()).layer(2,
                    new ActivationLayer.Builder()
                            .activation(Activation.RELU).build())
            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(10).nOut(10)
                    .build())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    network.setInput(next.getFeatures());
    INDArray activationsActual = network.output(next.getFeatures());
    assertEquals(10, activationsActual.shape()[1], 1e-2);

    network.fit(next);
    INDArray actualGammaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.GAMMA);
    INDArray actualBetaParam = network.getLayer(1).getParam(BatchNormalizationParamInitializer.BETA);
    assertTrue(actualGammaParam != null);
    assertTrue(actualBetaParam != null);
}
 
Example #29
Source File: Gan4Exemple.java    From dl4j-tutorials with MIT License 5 votes vote down vote up
private static void discInit() throws IOException {
    discUpdater = new RmsProp(drate);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(1234)
            .updater(new RmsProp(1e-3))
            .weightInit(WeightInit.XAVIER)
            .list()
            .layer(new DenseLayer.Builder()
                    .nIn(width * height)
                    .nOut(1024)
                    .activation(Activation.RELU)
                    .build())
            .layer(new DenseLayer.Builder()
                    .nIn(1024)
                    .nOut(512)
                    .activation(Activation.RELU)
                    .build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nIn(512)
                    .nOut(2)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.feedForward(width * height))
            .backprop(true).pretrain(false).build();

    discNet = new MultiLayerNetwork(conf);
    discNet.init();
    discNet.setListeners(new ScoreIterationListener(10));

}
 
Example #30
Source File: TestEarlyStoppingSpark.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testListeners() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd()).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(5));


    JavaRDD<DataSet> irisData = getIris();

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES))
                                    .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc()))
                                    .modelSaver(saver).build();

    LoggingEarlyStoppingListener listener = new LoggingEarlyStoppingListener();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new SparkEarlyStoppingTrainer(
                    getContext().sc(), new ParameterAveragingTrainingMaster(true,
                                    Runtime.getRuntime().availableProcessors(), 1, 10, 1, 0),
                    esConf, net, irisData);
    trainer.setListener(listener);

    trainer.fit();

    assertEquals(1, listener.onStartCallCount);
    assertEquals(5, listener.onEpochCallCount);
    assertEquals(1, listener.onCompletionCallCount);
}