Java Code Examples for org.nd4j.linalg.activations.Activation

The following examples show how to use org.nd4j.linalg.activations.Activation. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: deeplearning4j   Source File: LayerConfigValidationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCompGraphNullLayer() {
    ComputationGraphConfiguration.GraphBuilder gb = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.01))
            .seed(42).miniBatch(false).l1(0.2).l2(0.2)
            /* Graph Builder */
            .updater(Updater.RMSPROP).graphBuilder().addInputs("in")
            .addLayer("L" + 1,
                    new GravesLSTM.Builder().nIn(20).updater(Updater.RMSPROP).nOut(10)
                            .weightInit(WeightInit.XAVIER)
                            .dropOut(0.4).l1(0.3).activation(Activation.SIGMOID).build(),
                    "in")
            .addLayer("output",
                    new RnnOutputLayer.Builder().nIn(20).nOut(10).activation(Activation.SOFTMAX)
                            .weightInit(WeightInit.RELU_UNIFORM).build(),
                    "L" + 1)
            .setOutputs("output");
    ComputationGraphConfiguration conf = gb.build();
    ComputationGraph cg = new ComputationGraph(conf);
    cg.init();
}
 
Example 2
Source Project: deeplearning4j   Source File: ConvolutionLayerSetupTest.java    License: Apache License 2.0 6 votes vote down vote up
public MultiLayerConfiguration.Builder incompleteLRN() {
    MultiLayerConfiguration.Builder builder =
                    new NeuralNetConfiguration.Builder().seed(3)
                                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(2, new LocalResponseNormalization.Builder().build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                            LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2)
                                            .activation(Activation.SOFTMAX).build());
    return builder;
}
 
Example 3
Source Project: deeplearning4j   Source File: ConvDataFormatTests.java    License: Apache License 2.0 6 votes vote down vote up
private MultiLayerNetwork getDepthwiseConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new DepthwiseConvolution2D.Builder()
                .depthMultiplier(2)
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .dataFormat(format)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    } else {
        return getNetWithLayer(new DepthwiseConvolution2D.Builder()
                .depthMultiplier(2)
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    }
}
 
Example 4
@Test
public void testGraphOutputIterators(){

    DataSet all = new IrisDataSetIterator(150,150).next();
    DataSetIterator iter = new IrisDataSetIterator(5,150);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .graphBuilder()
            .addInputs("in")
            .layer("layer", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in")
            .setOutputs("layer")
            .build();
    ComputationGraph cg = new ComputationGraph(conf);
    cg.init();


    INDArray outAll = cg.outputSingle(all.getFeatures());
    INDArray outIter = cg.outputSingle(iter);

    assertEquals(outAll, outIter);
}
 
Example 5
Source Project: deeplearning4j   Source File: ModelSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidLoading1() throws Exception {
    ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder()
            .graphBuilder().addInputs("in")
            .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in")
            .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nIn(2).nOut(3).build(),
                    "dense")
            .setOutputs("out").build();

    ComputationGraph cg = new ComputationGraph(config);
    cg.init();

    File tempFile = tempDir.newFile();

    ModelSerializer.writeModel(cg, tempFile, true);

    try {
        ModelSerializer.restoreMultiLayerNetwork(tempFile);
        fail();
    } catch (Exception e){
        String msg = e.getMessage();
        assertTrue(msg, msg.contains("JSON") && msg.contains("restoreComputationGraph"));
    }
}
 
Example 6
Source Project: deeplearning4j   Source File: TestMultiLayerSpace.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGlobalPoolingBasic() {

    MultiLayerConfiguration expected = new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list()
                    .layer(0, new GravesLSTM.Builder().nIn(10).nOut(10).build())
                    .layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.SUM).pnorm(7).build())
                    .layer(2, new OutputLayer.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5).build())
                    .build();

    MultiLayerSpace mls =
                    new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345)
                                    .addLayer(new GravesLSTMLayerSpace.Builder().nIn(10).nOut(10).build())
                                    .addLayer(new GlobalPoolingLayerSpace.Builder().poolingType(PoolingType.SUM)
                                                    .pNorm(7).build())
                                    .addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT)
                                            .activation(Activation.SOFTMAX)
                                                    .nIn(10).nOut(5).build())
                                    .build();

    int nParams = mls.numParameters();
    assertEquals(0, nParams);

    MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration();

    assertEquals(expected, conf);
}
 
Example 7
Source Project: deeplearning4j   Source File: TestUpdaters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDivisionByMinibatch1(){
    //No batch norm - should be single INDArray equal to flattened gradient view

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    net.fit(Nd4j.create(1,10), Nd4j.create(1,10));

    BaseMultiLayerUpdater u = (BaseMultiLayerUpdater) net.getUpdater();
    List<INDArray> l = u.getGradientsForMinibatchDivision();
    assertNotNull(l);
    assertEquals(1, l.size());

    INDArray arr = l.get(0);
    assertEquals(3 * (10 * 10 + 10), arr.length());
    assertEquals(net.getFlattenedGradients(), arr);
}
 
Example 8
Source Project: deeplearning4j   Source File: DenseTest.java    License: Apache License 2.0 6 votes vote down vote up
private static MultiLayerNetwork getDenseMLNConfig(boolean backprop, boolean pretrain) {
    int numInputs = 4;
    int outputNum = 3;
    long seed = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
                    .updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    return model;

}
 
Example 9
Source Project: deeplearning4j   Source File: TestVertxUI.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUICompGraph() {

    StatsStorage ss = new InMemoryStatsStorage();

    UIServer uiServer = UIServer.getInstance();
    uiServer.attach(ss);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
                    .addLayer("L0", new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build(),
                                    "in")
                    .addLayer("L1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(4).nOut(3).build(), "L0")
                    .setOutputs("L1").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    net.setListeners(new StatsListener(ss), new ScoreIterationListener(1));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    for (int i = 0; i < 100; i++) {
        net.fit(iter);
    }
}
 
Example 10
Source Project: deeplearning4j   Source File: SqueezeNet.java    License: Apache License 2.0 6 votes vote down vote up
private String fireModule(ComputationGraphConfiguration.GraphBuilder graphBuilder, int fireId, int squeeze, int expand, String input) {
    String prefix = "fire"+fireId;

    graphBuilder
            .addLayer(prefix+"_sq1x1", new ConvolutionLayer.Builder(1, 1).nOut(squeeze)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), input)
            .addLayer(prefix+"_relu_sq1x1", new ActivationLayer(Activation.RELU), prefix+"_sq1x1")

            .addLayer(prefix+"_exp1x1", new ConvolutionLayer.Builder(1, 1).nOut(expand)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), prefix+"_relu_sq1x1")
            .addLayer(prefix+"_relu_exp1x1", new ActivationLayer(Activation.RELU), prefix+"_exp1x1")

            .addLayer(prefix+"_exp3x3", new ConvolutionLayer.Builder(3,3).nOut(expand)
                    .convolutionMode(ConvolutionMode.Same)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), prefix+"_relu_sq1x1")
            .addLayer(prefix+"_relu_exp3x3", new ActivationLayer(Activation.RELU), prefix+"_exp3x3")

            .addVertex(prefix, new MergeVertex(), prefix+"_relu_exp1x1", prefix+"_relu_exp3x3");

    return prefix;
}
 
Example 11
private static ComputationGraphConfiguration getGraphConfCNN(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "in")
                    .addLayer("1", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "0")
                    .addLayer("2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10)
                                    .build(), "1")
                    .setOutputs("2").setInputTypes(InputType.convolutional(10, 10, 3))
                    .build();
    return conf;
}
 
Example 12
@Test
public void testBidirectionalGraphSummary() {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
            .addLayer("bidirectional",
                    new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()),
                    "in")
            .addLayer("out", new RnnOutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build(), "bidirectional")
            .setOutputs("out")
            .setInputTypes(new InputType.InputTypeRecurrent(10, 12))
            .build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    graph.summary();
}
 
Example 13
@Test
    public void testLSTMWithSubset() {
        Nd4j.getRandom().setSeed(1234);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(1234)
                        .dataType(DataType.DOUBLE)
                        .weightInit(new NormalDistribution(0, 1))
                        .updater(new NoOp()).graphBuilder().addInputs("input").setOutputs("out")
                        .addLayer("lstm1", new LSTM.Builder().nIn(3).nOut(6).activation(Activation.TANH).build(),
                                        "input")
                        .addVertex("subset", new SubsetVertex(0, 2), "lstm1")
                        .addLayer("out", new RnnOutputLayer.Builder().nIn(3).nOut(2).activation(Activation.SOFTMAX)
                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "subset")
                        .build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        Random r = new Random(12345);
        INDArray input = Nd4j.rand(new int[] {2, 3, 4});
        INDArray labels = TestUtils.randomOneHotTimeSeries(2, 2, 4);

        if (PRINT_RESULTS) {
            System.out.println("testLSTMWithSubset()");
//            for (int j = 0; j < graph.getNumLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                .labels(new INDArray[]{labels}));

        String msg = "testLSTMWithSubset()";
        assertTrue(msg, gradOK);
        TestUtils.testModelSerialization(graph);
    }
 
Example 14
Source Project: deeplearning4j   Source File: BackPropMLPTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
    public void testMLP() {
        //Simple mini-batch test with multiple hidden layers
        MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] {5, 4, 3}, Activation.SIGMOID);
//        System.out.println(conf);
        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();
        DataSetIterator iter = new IrisDataSetIterator(10, 100);

        while (iter.hasNext()) {
            network.fit(iter.next());
        }
    }
 
Example 15
@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .graphBuilder().addInputs("in")
            .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build(), "in")
            .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                            LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                            .build(),
                    "0")
            .setOutputs("1").build();


    ComputationGraph network = new ComputationGraph(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getConfiguration().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getConfiguration().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    ComputationGraph net = ModelSerializer.restoreComputationGraph(bais, true);
    assertEquals(7, net.getConfiguration().getIterationCount());
}
 
Example 16
@Test
public void testCompGraphDropoutOutputLayers2(){
    //https://github.com/deeplearning4j/deeplearning4j/issues/6326
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .dropOut(0.8)
            .graphBuilder()
            .addInputs("in1", "in2")
            .addVertex("merge", new MergeVertex(), "in1", "in2")
            .addLayer("dense",
                    new DenseLayer.Builder()
                            .nIn(10).nOut(5)
                            .activation(Activation.TANH)
                            .dropOut(new GaussianNoise(0.05))
                            .build(),"merge")
            .addLayer("out1",
                    new OutputLayer.Builder().activation(Activation.SOFTMAX)
                            .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5)
                            .nOut(6).build(),
                    "dense")
            .addLayer("out2",
                    new OutputLayer.Builder().activation(Activation.SOFTMAX)
                            .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5)
                            .nOut(4).build(),
                    "dense")
            .setOutputs("out1", "out2").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    INDArray[] features = new INDArray[]{Nd4j.create(1, 5), Nd4j.create(1, 5)};
    INDArray[] labels = new INDArray[]{Nd4j.create(1, 6), Nd4j.create(1, 4)};
    MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(features, labels);
    net.fit(mds);
}
 
Example 17
Source Project: deeplearning4j   Source File: MultiLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLayerNames() {
    int nIn = 10;
    int nOut = 40;

    List<String> layerNameList = new ArrayList<>();
    layerNameList.add("dnn1");
    layerNameList.add("dnn2");
    layerNameList.add("dnn3");

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .updater(new Sgd(0.1)).list()
                    .layer(0, new DenseLayer.Builder().name("dnn1").nIn(nIn).nOut(20).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(1, new DenseLayer.Builder().name("dnn2").nIn(20).nOut(30).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(2, new DenseLayer.Builder().name("dnn3").nIn(30).nOut(nOut)
                                    .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    assertEquals(layerNameList.get(0), net.getLayer(0).conf().getLayer().getLayerName());
    assertEquals(layerNameList, net.getLayerNames());
    BaseLayer b = (BaseLayer) net.getLayer(layerNameList.get(2)).conf().getLayer();
    assertEquals("softmax", b.getActivationFn().toString());
}
 
Example 18
@Test
public void testJSONBasic2() {
    ComputationGraphConfiguration conf =
            new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .graphBuilder().addInputs("input")
                    .addLayer("cnn1",
                            new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5)
                                    .build(),
                            "input")
                    .addLayer("cnn2",
                            new ConvolutionLayer.Builder(2, 2).stride(2, 2).nIn(1).nOut(5)
                                    .build(),
                            "input")
                    .addLayer("max1",
                            new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                    .kernelSize(2, 2).build(),
                            "cnn1", "cnn2")
                    .addLayer("dnn1", new DenseLayer.Builder().nOut(7).build(), "max1")
                    .addLayer("max2", new SubsamplingLayer.Builder().build(), "max1")
                    .addLayer("output", new OutputLayer.Builder().nIn(7).nOut(10).activation(Activation.SOFTMAX).build(), "dnn1",
                            "max2")
                    .setOutputs("output")
                    .inputPreProcessor("cnn1", new FeedForwardToCnnPreProcessor(32, 32, 3))
                    .inputPreProcessor("cnn2", new FeedForwardToCnnPreProcessor(32, 32, 3))
                    .inputPreProcessor("dnn1", new CnnToFeedForwardPreProcessor(8, 8, 5))
                    .build();

    String json = conf.toJson();
    ComputationGraphConfiguration conf2 = ComputationGraphConfiguration.fromJson(json);

    assertEquals(json, conf2.toJson());
    assertEquals(conf, conf2);
}
 
Example 19
Source Project: deeplearning4j   Source File: MultiLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSetScoreCNN() {

    int miniBatch = 3;
    int depth = 2;
    int width = 3;
    int height = 3;
    int nOut = 2;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .seed(12345L).list().layer(0, new ConvolutionLayer.Builder(2, 2).nOut(1).build())
                    .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(2).build())
                    .setInputType(InputType.convolutionalFlat(height, width, depth))
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);
    INDArray input = Nd4j.rand(miniBatch, depth * width * height);
    INDArray labels = Nd4j.create(miniBatch, nOut);
    for (int i = 0; i < miniBatch; i++) {
        labels.putScalar(new int[] {i, r.nextInt(nOut)}, 1.0);
    }

    double score = net.score(new DataSet(input, labels));
}
 
Example 20
@Test
public void testDisconnectedVertex(){

    for(boolean allowDisconnected : new boolean[]{false, true}) {
        try {
            ComputationGraphConfiguration.GraphBuilder b = new NeuralNetConfiguration.Builder()
                    .graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new DenseLayer.Builder().activation(Activation.SIGMOID).nOut(8).build(), "in")
                    .addLayer("1", new DenseLayer.Builder().activation(Activation.SIGMOID).nOut(8).build(), "in") //Disconnected
                    .addLayer("O", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(10).build(), "0")
                    .setOutputs("O")
                    .setInputTypes(InputType.feedForward(8));

            if(allowDisconnected){
                b.allowDisconnected(true).build();  //No exception
            } else {
                b.build();  //Expect exception here
                fail("Expected exception for disconnected vertex");
            }


        } catch (Exception e) {
            log.error("",e);
            if(allowDisconnected){
                fail("No exception expected");
            } else {
                String msg = e.getMessage().toLowerCase();
                assertTrue(msg.contains("disconnected"));
            }
        }
    }
}
 
Example 21
Source Project: deeplearning4j   Source File: BaseLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
public MultiLayerNetwork configureMultiLayer() {
    int nIn = 2;
    int nOut = 2;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(nOut).build())
                    .layer(1, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    return net;
}
 
Example 22
Source Project: dl4j-tutorials   Source File: MultiRegression.java    License: MIT License 5 votes vote down vote up
public static void main(String[] args){

        //Generate the training data
        DataSetIterator iterator = getTrainingData(batchSize,rng);

        //Create the network
        int numInput = 2;
        int numOutputs = 1;
        MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
                .seed(seed)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .weightInit(WeightInit.XAVIER)
                .updater(new Sgd(learningRate))
                .list()
                .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE)
                        .activation(Activation.IDENTITY)
                        .nIn(numInput).nOut(numOutputs).build())
                .pretrain(false).backprop(true).build()
        );
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        for( int i=0; i<nEpochs; i++ ){
            iterator.reset();
            net.fit(iterator);
        }

        final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 });
        INDArray out = net.output(input, false);
        System.out.println(out);

    }
 
Example 23
Source Project: deeplearning4j   Source File: GravesLSTMTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGateActivationFnsSanityCheck() {
    for (String gateAfn : new String[] {"sigmoid", "hardsigmoid"}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .seed(12345).list()
                        .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder()
                                        .gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2)
                                        .build())
                        .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder()
                                        .lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2)
                                        .activation(Activation.TANH).build())
                        .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesLSTM) net.getLayer(0).conf().getLayer())
                        .getGateActivationFn().toString());

        INDArray in = Nd4j.rand(new int[] {3, 2, 5});
        INDArray labels = Nd4j.rand(new int[] {3, 2, 5});

        net.fit(in, labels);
    }
}
 
Example 24
Source Project: deeplearning4j   Source File: DataSetIteratorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
    public void testLfwModel() throws Exception {
        final int numRows = 28;
        final int numColumns = 28;
        int numChannels = 3;
        int outputNum = LFWLoader.NUM_LABELS;
        int numSamples = LFWLoader.NUM_IMAGES;
        int batchSize = 2;
        int seed = 123;
        int listenerFreq = 1;

        LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples,
                        new int[] {numRows, numColumns, numChannels}, outputNum, false, true, 1.0, new Random(seed));

        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6)
                                        .weightInit(WeightInit.XAVIER).activation(Activation.RELU).build())
                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                        .stride(1, 1).build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                        .build())
                        .setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels))
                        ;

        MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
        model.init();

        model.setListeners(new ScoreIterationListener(listenerFreq));

        model.fit(lfw.next());

        DataSet dataTest = lfw.next();
        INDArray output = model.output(dataTest.getFeatures());
        Evaluation eval = new Evaluation(outputNum);
        eval.eval(dataTest.getLabels(), output);
//        System.out.println(eval.stats());
    }
 
Example 25
@Test
public void testListOverloads() {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build())
                    .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DenseLayer dl = (DenseLayer) conf.getConf(0).getLayer();
    assertEquals(3, dl.getNIn());
    assertEquals(4, dl.getNOut());
    OutputLayer ol = (OutputLayer) conf.getConf(1).getLayer();
    assertEquals(4, ol.getNIn());
    assertEquals(5, ol.getNOut());

    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build())
                    .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345)
                    .list(new DenseLayer.Builder().nIn(3).nOut(4).build(),
                                    new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net3 = new MultiLayerNetwork(conf3);
    net3.init();


    assertEquals(conf, conf2);
    assertEquals(conf, conf3);
}
 
Example 26
Source Project: deeplearning4j   Source File: KerasActivationUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Map Keras to DL4J activation functions.
 *
 * @param conf Keras layer configuration
 * @param kerasActivation String containing Keras activation function name
 * @return Activation enum value containing DL4J activation function name
 */
public static Activation mapToActivation(String kerasActivation, KerasLayerConfiguration conf)
        throws UnsupportedKerasConfigurationException {
    Activation dl4jActivation;
    if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTMAX())) {
        dl4jActivation = Activation.SOFTMAX;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTPLUS())) {
        dl4jActivation = Activation.SOFTPLUS;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTSIGN())) {
        dl4jActivation = Activation.SOFTSIGN;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_RELU())) {
        dl4jActivation = Activation.RELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_RELU6())) {
        dl4jActivation = Activation.RELU6;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_ELU())) {
        dl4jActivation = Activation.ELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SELU())) {
        dl4jActivation = Activation.SELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_TANH())) {
        dl4jActivation = Activation.TANH;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SIGMOID())) {
        dl4jActivation = Activation.SIGMOID;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_HARD_SIGMOID())) {
        dl4jActivation = Activation.HARDSIGMOID;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_LINEAR())) {
        dl4jActivation = Activation.IDENTITY;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SWISH())) {
        dl4jActivation = Activation.SWISH;
    } else {
        throw new UnsupportedKerasConfigurationException(
                "Unknown Keras activation function " + kerasActivation);
    }
    return dl4jActivation;
}
 
Example 27
Source Project: deeplearning4j   Source File: KerasActivationUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get activation enum value from Keras layer configuration.
 *
 * @param layerConfig dictionary containing Keras layer configuration
 * @return DL4J activation enum value
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public static Activation getActivationFromConfig(Map<String, Object> layerConfig, KerasLayerConfiguration conf)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    if (!innerConfig.containsKey(conf.getLAYER_FIELD_ACTIVATION()))
        throw new InvalidKerasConfigurationException("Keras layer is missing "
                + conf.getLAYER_FIELD_ACTIVATION() + " field");
    return mapToActivation((String) innerConfig.get(conf.getLAYER_FIELD_ACTIVATION()), conf);
}
 
Example 28
Source Project: deeplearning4j   Source File: TestEarlyStoppingCompGraph.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                    new ScoreImprovementEpochTerminationCondition(5))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                    .scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 
Example 29
Source Project: deeplearning4j   Source File: WorkspaceTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
    public void testWithPreprocessorsCG() {
        //https://github.com/deeplearning4j/deeplearning4j/issues/4347
        //Cause for the above issue was layerVertex.setInput() applying the preprocessor, with the result
        // not being detached properly from the workspace...

        for (WorkspaceMode wm : WorkspaceMode.values()) {
            System.out.println(wm);
            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .trainingWorkspaceMode(wm)
                    .inferenceWorkspaceMode(wm)
                    .graphBuilder()
                    .addInputs("in")
                    .addLayer("e", new GravesLSTM.Builder().nIn(10).nOut(5).build(), new DupPreProcessor(), "in")
//                .addLayer("e", new GravesLSTM.Builder().nIn(10).nOut(5).build(), "in")    //Note that no preprocessor is OK
                    .addLayer("rnn", new GravesLSTM.Builder().nIn(5).nOut(8).build(), "e")
                    .addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
                            .activation(Activation.SIGMOID).nOut(3).build(), "rnn")
                    .setInputTypes(InputType.recurrent(10))
                    .setOutputs("out")
                    .build();

            ComputationGraph cg = new ComputationGraph(conf);
            cg.init();


            INDArray[] input = new INDArray[]{Nd4j.zeros(1, 10, 5)};

            for (boolean train : new boolean[]{false, true}) {
                cg.clear();
                cg.feedForward(input, train);
            }

            cg.setInputs(input);
            cg.setLabels(Nd4j.rand(new int[]{1, 3, 5}));
            cg.computeGradientAndScore();
        }
    }
 
Example 30
Source Project: deeplearning4j   Source File: TestVertxUI.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUIMultipleSessions() throws Exception {

    for (int session = 0; session < 3; session++) {

        StatsStorage ss = new InMemoryStatsStorage();

        UIServer uiServer = UIServer.getInstance();
        uiServer.attach(ss);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build())
                .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                        .activation(Activation.SOFTMAX).nIn(4).nOut(3).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        net.setListeners(new StatsListener(ss, 1), new ScoreIterationListener(1));

        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        for (int i = 0; i < 20; i++) {
            net.fit(iter);
            Thread.sleep(100);
        }
    }
}