org.nd4j.linalg.activations.Activation Java Examples

The following examples show how to use org.nd4j.linalg.activations.Activation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ComputationGraphConfigurationTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testBidirectionalGraphSummary() {
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
            .addLayer("bidirectional",
                    new Bidirectional(new LSTM.Builder().activation(Activation.TANH).nOut(10).build()),
                    "in")
            .addLayer("out", new RnnOutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build(), "bidirectional")
            .setOutputs("out")
            .setInputTypes(new InputType.InputTypeRecurrent(10, 12))
            .build();

    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();
    graph.summary();
}
 
Example #2
Source File: LayerConfigValidationTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompGraphNullLayer() {
    ComputationGraphConfiguration.GraphBuilder gb = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Sgd(0.01))
            .seed(42).miniBatch(false).l1(0.2).l2(0.2)
            /* Graph Builder */
            .updater(Updater.RMSPROP).graphBuilder().addInputs("in")
            .addLayer("L" + 1,
                    new GravesLSTM.Builder().nIn(20).updater(Updater.RMSPROP).nOut(10)
                            .weightInit(WeightInit.XAVIER)
                            .dropOut(0.4).l1(0.3).activation(Activation.SIGMOID).build(),
                    "in")
            .addLayer("output",
                    new RnnOutputLayer.Builder().nIn(20).nOut(10).activation(Activation.SOFTMAX)
                            .weightInit(WeightInit.RELU_UNIFORM).build(),
                    "L" + 1)
            .setOutputs("output");
    ComputationGraphConfiguration conf = gb.build();
    ComputationGraph cg = new ComputationGraph(conf);
    cg.init();
}
 
Example #3
Source File: ConvolutionLayerSetupTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public MultiLayerConfiguration.Builder incompleteLRN() {
    MultiLayerConfiguration.Builder builder =
                    new NeuralNetConfiguration.Builder().seed(3)
                                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(2, new LocalResponseNormalization.Builder().build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                            LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2)
                                            .activation(Activation.SOFTMAX).build());
    return builder;
}
 
Example #4
Source File: ConvDataFormatTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private MultiLayerNetwork getDepthwiseConv2dNet(CNN2DFormat format, boolean setOnLayerAlso, ConvolutionMode cm) {
    if (setOnLayerAlso) {
        return getNetWithLayer(new DepthwiseConvolution2D.Builder()
                .depthMultiplier(2)
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .dataFormat(format)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    } else {
        return getNetWithLayer(new DepthwiseConvolution2D.Builder()
                .depthMultiplier(2)
                .kernelSize(3, 3)
                .stride(2, 2)
                .activation(Activation.TANH)
                .nOut(3)
                .helperAllowFallback(false)
                .build(), format, cm, null);
    }
}
 
Example #5
Source File: TestComputationGraphNetwork.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testGraphOutputIterators(){

    DataSet all = new IrisDataSetIterator(150,150).next();
    DataSetIterator iter = new IrisDataSetIterator(5,150);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .graphBuilder()
            .addInputs("in")
            .layer("layer", new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).build(), "in")
            .setOutputs("layer")
            .build();
    ComputationGraph cg = new ComputationGraph(conf);
    cg.init();


    INDArray outAll = cg.outputSingle(all.getFeatures());
    INDArray outIter = cg.outputSingle(iter);

    assertEquals(outAll, outIter);
}
 
Example #6
Source File: ModelSerializerTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidLoading1() throws Exception {
    ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder()
            .graphBuilder().addInputs("in")
            .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in")
            .addLayer("out",new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nIn(2).nOut(3).build(),
                    "dense")
            .setOutputs("out").build();

    ComputationGraph cg = new ComputationGraph(config);
    cg.init();

    File tempFile = tempDir.newFile();

    ModelSerializer.writeModel(cg, tempFile, true);

    try {
        ModelSerializer.restoreMultiLayerNetwork(tempFile);
        fail();
    } catch (Exception e){
        String msg = e.getMessage();
        assertTrue(msg, msg.contains("JSON") && msg.contains("restoreComputationGraph"));
    }
}
 
Example #7
Source File: TestMultiLayerSpace.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testGlobalPoolingBasic() {

    MultiLayerConfiguration expected = new NeuralNetConfiguration.Builder().updater(new Sgd(0.005)).seed(12345).list()
                    .layer(0, new GravesLSTM.Builder().nIn(10).nOut(10).build())
                    .layer(1, new GlobalPoolingLayer.Builder().poolingType(PoolingType.SUM).pnorm(7).build())
                    .layer(2, new OutputLayer.Builder().lossFunction(LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(10).nOut(5).build())
                    .build();

    MultiLayerSpace mls =
                    new MultiLayerSpace.Builder().updater(new Sgd(0.005)).seed(12345)
                                    .addLayer(new GravesLSTMLayerSpace.Builder().nIn(10).nOut(10).build())
                                    .addLayer(new GlobalPoolingLayerSpace.Builder().poolingType(PoolingType.SUM)
                                                    .pNorm(7).build())
                                    .addLayer(new OutputLayerSpace.Builder().lossFunction(LossFunction.MCXENT)
                                            .activation(Activation.SOFTMAX)
                                                    .nIn(10).nOut(5).build())
                                    .build();

    int nParams = mls.numParameters();
    assertEquals(0, nParams);

    MultiLayerConfiguration conf = mls.getValue(new double[0]).getMultiLayerConfiguration();

    assertEquals(expected, conf);
}
 
Example #8
Source File: DenseTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private static MultiLayerNetwork getDenseMLNConfig(boolean backprop, boolean pretrain) {
    int numInputs = 4;
    int outputNum = 3;
    long seed = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
                    .updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    return model;

}
 
Example #9
Source File: TestVertxUI.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testUICompGraph() {

    StatsStorage ss = new InMemoryStatsStorage();

    UIServer uiServer = UIServer.getInstance();
    uiServer.attach(ss);

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder().addInputs("in")
                    .addLayer("L0", new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build(),
                                    "in")
                    .addLayer("L1", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(4).nOut(3).build(), "L0")
                    .setOutputs("L1").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    net.setListeners(new StatsListener(ss), new ScoreIterationListener(1));

    DataSetIterator iter = new IrisDataSetIterator(150, 150);

    for (int i = 0; i < 100; i++) {
        net.fit(iter);
    }
}
 
Example #10
Source File: SqueezeNet.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private String fireModule(ComputationGraphConfiguration.GraphBuilder graphBuilder, int fireId, int squeeze, int expand, String input) {
    String prefix = "fire"+fireId;

    graphBuilder
            .addLayer(prefix+"_sq1x1", new ConvolutionLayer.Builder(1, 1).nOut(squeeze)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), input)
            .addLayer(prefix+"_relu_sq1x1", new ActivationLayer(Activation.RELU), prefix+"_sq1x1")

            .addLayer(prefix+"_exp1x1", new ConvolutionLayer.Builder(1, 1).nOut(expand)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), prefix+"_relu_sq1x1")
            .addLayer(prefix+"_relu_exp1x1", new ActivationLayer(Activation.RELU), prefix+"_exp1x1")

            .addLayer(prefix+"_exp3x3", new ConvolutionLayer.Builder(3,3).nOut(expand)
                    .convolutionMode(ConvolutionMode.Same)
                    .cudnnAlgoMode(cudnnAlgoMode).build(), prefix+"_relu_sq1x1")
            .addLayer(prefix+"_relu_exp3x3", new ActivationLayer(Activation.RELU), prefix+"_exp3x3")

            .addVertex(prefix, new MergeVertex(), prefix+"_relu_exp1x1", prefix+"_relu_exp3x3");

    return prefix;
}
 
Example #11
Source File: TestCompareParameterAveragingSparkVsSingleMachine.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private static ComputationGraphConfiguration getGraphConfCNN(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "in")
                    .addLayer("1", new ConvolutionLayer.Builder().nOut(3).kernelSize(2, 2).stride(1, 1)
                                    .padding(0, 0).activation(Activation.TANH).build(), "0")
                    .addLayer("2", new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nOut(10)
                                    .build(), "1")
                    .setOutputs("2").setInputTypes(InputType.convolutional(10, 10, 3))
                    .build();
    return conf;
}
 
Example #12
Source File: TestComputationGraphNetwork.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .graphBuilder().addInputs("in")
            .addLayer("0", new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build(), "in")
            .addLayer("1", new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                            LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                            .build(),
                    "0")
            .setOutputs("1").build();


    ComputationGraph network = new ComputationGraph(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getConfiguration().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getConfiguration().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getConfiguration().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    ComputationGraph net = ModelSerializer.restoreComputationGraph(bais, true);
    assertEquals(7, net.getConfiguration().getIterationCount());
}
 
Example #13
Source File: KerasActivationUtils.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * Map Keras to DL4J activation functions.
 *
 * @param conf Keras layer configuration
 * @param kerasActivation String containing Keras activation function name
 * @return Activation enum value containing DL4J activation function name
 */
public static Activation mapToActivation(String kerasActivation, KerasLayerConfiguration conf)
        throws UnsupportedKerasConfigurationException {
    Activation dl4jActivation;
    if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTMAX())) {
        dl4jActivation = Activation.SOFTMAX;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTPLUS())) {
        dl4jActivation = Activation.SOFTPLUS;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SOFTSIGN())) {
        dl4jActivation = Activation.SOFTSIGN;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_RELU())) {
        dl4jActivation = Activation.RELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_RELU6())) {
        dl4jActivation = Activation.RELU6;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_ELU())) {
        dl4jActivation = Activation.ELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SELU())) {
        dl4jActivation = Activation.SELU;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_TANH())) {
        dl4jActivation = Activation.TANH;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SIGMOID())) {
        dl4jActivation = Activation.SIGMOID;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_HARD_SIGMOID())) {
        dl4jActivation = Activation.HARDSIGMOID;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_LINEAR())) {
        dl4jActivation = Activation.IDENTITY;
    } else if (kerasActivation.equals(conf.getKERAS_ACTIVATION_SWISH())) {
        dl4jActivation = Activation.SWISH;
    } else {
        throw new UnsupportedKerasConfigurationException(
                "Unknown Keras activation function " + kerasActivation);
    }
    return dl4jActivation;
}
 
Example #14
Source File: TestSparkComputationGraph.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public static ComputationGraph getBasicNetIris2Class() {

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
                        .graphBuilder().addInputs("in")
                        .addLayer("l0", new DenseLayer.Builder().nIn(4).nOut(10).build(), "in")
                        .addLayer("l1", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                        .activation(Activation.SOFTMAX).nIn(10).nOut(2).build(), "l0")
                        .setOutputs("l1").build();

        ComputationGraph cg = new ComputationGraph(conf);
        cg.init();

        return cg;
    }
 
Example #15
Source File: MultiLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testLayerNames() {
    int nIn = 10;
    int nOut = 40;

    List<String> layerNameList = new ArrayList<>();
    layerNameList.add("dnn1");
    layerNameList.add("dnn2");
    layerNameList.add("dnn3");

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .updater(new Sgd(0.1)).list()
                    .layer(0, new DenseLayer.Builder().name("dnn1").nIn(nIn).nOut(20).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(1, new DenseLayer.Builder().name("dnn2").nIn(20).nOut(30).activation(Activation.RELU)
                                    .weightInit(WeightInit.XAVIER).build())
                    .layer(2, new DenseLayer.Builder().name("dnn3").nIn(30).nOut(nOut)
                                    .activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    assertEquals(layerNameList.get(0), net.getLayer(0).conf().getLayer().getLayerName());
    assertEquals(layerNameList, net.getLayerNames());
    BaseLayer b = (BaseLayer) net.getLayer(layerNameList.get(2)).conf().getLayer();
    assertEquals("softmax", b.getActivationFn().toString());
}
 
Example #16
Source File: TestVertxUI.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testUIMultipleSessions() throws Exception {

    for (int session = 0; session < 3; session++) {

        StatsStorage ss = new InMemoryStatsStorage();

        UIServer uiServer = UIServer.getInstance();
        uiServer.attach(ss);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                .layer(0, new DenseLayer.Builder().activation(Activation.TANH).nIn(4).nOut(4).build())
                .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MCXENT)
                        .activation(Activation.SOFTMAX).nIn(4).nOut(3).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        net.setListeners(new StatsListener(ss, 1), new ScoreIterationListener(1));

        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        for (int i = 0; i < 20; i++) {
            net.fit(iter);
            Thread.sleep(100);
        }
    }
}
 
Example #17
Source File: TestComputationGraphNetwork.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCompGraphDropoutOutputLayers2(){
    //https://github.com/deeplearning4j/deeplearning4j/issues/6326
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
            .dropOut(0.8)
            .graphBuilder()
            .addInputs("in1", "in2")
            .addVertex("merge", new MergeVertex(), "in1", "in2")
            .addLayer("dense",
                    new DenseLayer.Builder()
                            .nIn(10).nOut(5)
                            .activation(Activation.TANH)
                            .dropOut(new GaussianNoise(0.05))
                            .build(),"merge")
            .addLayer("out1",
                    new OutputLayer.Builder().activation(Activation.SOFTMAX)
                            .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5)
                            .nOut(6).build(),
                    "dense")
            .addLayer("out2",
                    new OutputLayer.Builder().activation(Activation.SOFTMAX)
                            .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(5)
                            .nOut(4).build(),
                    "dense")
            .setOutputs("out1", "out2").build();

    ComputationGraph net = new ComputationGraph(conf);
    net.init();

    INDArray[] features = new INDArray[]{Nd4j.create(1, 5), Nd4j.create(1, 5)};
    INDArray[] labels = new INDArray[]{Nd4j.create(1, 6), Nd4j.create(1, 4)};
    MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet(features, labels);
    net.fit(mds);
}
 
Example #18
Source File: MultiLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSetScoreCNN() {

    int miniBatch = 3;
    int depth = 2;
    int width = 3;
    int height = 3;
    int nOut = 2;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .seed(12345L).list().layer(0, new ConvolutionLayer.Builder(2, 2).nOut(1).build())
                    .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nOut(2).build())
                    .setInputType(InputType.convolutionalFlat(height, width, depth))
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Nd4j.getRandom().setSeed(12345);
    Random r = new Random(12345);
    INDArray input = Nd4j.rand(miniBatch, depth * width * height);
    INDArray labels = Nd4j.create(miniBatch, nOut);
    for (int i = 0; i < miniBatch; i++) {
        labels.putScalar(new int[] {i, r.nextInt(nOut)}, 1.0);
    }

    double score = net.score(new DataSet(input, labels));
}
 
Example #19
Source File: GradientCheckTestsComputationGraph.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testLSTMWithSubset() {
        Nd4j.getRandom().setSeed(1234);
        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(1234)
                        .dataType(DataType.DOUBLE)
                        .weightInit(new NormalDistribution(0, 1))
                        .updater(new NoOp()).graphBuilder().addInputs("input").setOutputs("out")
                        .addLayer("lstm1", new LSTM.Builder().nIn(3).nOut(6).activation(Activation.TANH).build(),
                                        "input")
                        .addVertex("subset", new SubsetVertex(0, 2), "lstm1")
                        .addLayer("out", new RnnOutputLayer.Builder().nIn(3).nOut(2).activation(Activation.SOFTMAX)
                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "subset")
                        .build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        Random r = new Random(12345);
        INDArray input = Nd4j.rand(new int[] {2, 3, 4});
        INDArray labels = TestUtils.randomOneHotTimeSeries(2, 2, 4);

        if (PRINT_RESULTS) {
            System.out.println("testLSTMWithSubset()");
//            for (int j = 0; j < graph.getNumLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + graph.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(graph).inputs(new INDArray[]{input})
                .labels(new INDArray[]{labels}));

        String msg = "testLSTMWithSubset()";
        assertTrue(msg, gradOK);
        TestUtils.testModelSerialization(graph);
    }
 
Example #20
Source File: KerasActivationUtils.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * Get activation enum value from Keras layer configuration.
 *
 * @param layerConfig dictionary containing Keras layer configuration
 * @return DL4J activation enum value
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public static Activation getActivationFromConfig(Map<String, Object> layerConfig, KerasLayerConfiguration conf)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    if (!innerConfig.containsKey(conf.getLAYER_FIELD_ACTIVATION()))
        throw new InvalidKerasConfigurationException("Keras layer is missing "
                + conf.getLAYER_FIELD_ACTIVATION() + " field");
    return mapToActivation((String) innerConfig.get(conf.getLAYER_FIELD_ACTIVATION()), conf);
}
 
Example #21
Source File: WorkspaceTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testWithPreprocessorsCG() {
        //https://github.com/deeplearning4j/deeplearning4j/issues/4347
        //Cause for the above issue was layerVertex.setInput() applying the preprocessor, with the result
        // not being detached properly from the workspace...

        for (WorkspaceMode wm : WorkspaceMode.values()) {
            System.out.println(wm);
            ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
                    .trainingWorkspaceMode(wm)
                    .inferenceWorkspaceMode(wm)
                    .graphBuilder()
                    .addInputs("in")
                    .addLayer("e", new GravesLSTM.Builder().nIn(10).nOut(5).build(), new DupPreProcessor(), "in")
//                .addLayer("e", new GravesLSTM.Builder().nIn(10).nOut(5).build(), "in")    //Note that no preprocessor is OK
                    .addLayer("rnn", new GravesLSTM.Builder().nIn(5).nOut(8).build(), "e")
                    .addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MSE)
                            .activation(Activation.SIGMOID).nOut(3).build(), "rnn")
                    .setInputTypes(InputType.recurrent(10))
                    .setOutputs("out")
                    .build();

            ComputationGraph cg = new ComputationGraph(conf);
            cg.init();


            INDArray[] input = new INDArray[]{Nd4j.zeros(1, 10, 5)};

            for (boolean train : new boolean[]{false, true}) {
                cg.clear();
                cg.feedForward(input, train);
            }

            cg.setInputs(input);
            cg.setLabels(Nd4j.rand(new int[]{1, 3, 5}));
            cg.computeGradientAndScore();
        }
    }
 
Example #22
Source File: BaseLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public MultiLayerNetwork configureMultiLayer() {
    int nIn = 2;
    int nOut = 2;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(nOut).build())
                    .layer(1, new OutputLayer.Builder().nIn(nIn).nOut(nOut).activation(Activation.SOFTMAX).build()).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    return net;
}
 
Example #23
Source File: BackPropMLPTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testMLP() {
        //Simple mini-batch test with multiple hidden layers
        MultiLayerConfiguration conf = getIrisMLPSimpleConfig(new int[] {5, 4, 3}, Activation.SIGMOID);
//        System.out.println(conf);
        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();
        DataSetIterator iter = new IrisDataSetIterator(10, 100);

        while (iter.hasNext()) {
            network.fit(iter.next());
        }
    }
 
Example #24
Source File: MultiRegression.java    From dl4j-tutorials with MIT License 5 votes vote down vote up
public static void main(String[] args){

        //Generate the training data
        DataSetIterator iterator = getTrainingData(batchSize,rng);

        //Create the network
        int numInput = 2;
        int numOutputs = 1;
        MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
                .seed(seed)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .weightInit(WeightInit.XAVIER)
                .updater(new Sgd(learningRate))
                .list()
                .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE)
                        .activation(Activation.IDENTITY)
                        .nIn(numInput).nOut(numOutputs).build())
                .pretrain(false).backprop(true).build()
        );
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        for( int i=0; i<nEpochs; i++ ){
            iterator.reset();
            net.fit(iterator);
        }

        final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 });
        INDArray out = net.output(input, false);
        System.out.println(out);

    }
 
Example #25
Source File: MultiLayerNeuralNetConfigurationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testListOverloads() {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build())
                    .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DenseLayer dl = (DenseLayer) conf.getConf(0).getLayer();
    assertEquals(3, dl.getNIn());
    assertEquals(4, dl.getNOut());
    OutputLayer ol = (OutputLayer) conf.getConf(1).getLayer();
    assertEquals(4, ol.getNIn());
    assertEquals(5, ol.getNOut());

    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new DenseLayer.Builder().nIn(3).nOut(4).build())
                    .layer(1, new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net2 = new MultiLayerNetwork(conf2);
    net2.init();

    MultiLayerConfiguration conf3 = new NeuralNetConfiguration.Builder().seed(12345)
                    .list(new DenseLayer.Builder().nIn(3).nOut(4).build(),
                                    new OutputLayer.Builder().nIn(4).nOut(5).activation(Activation.SOFTMAX).build())
                    .build();
    MultiLayerNetwork net3 = new MultiLayerNetwork(conf3);
    net3.init();


    assertEquals(conf, conf2);
    assertEquals(conf, conf3);
}
 
Example #26
Source File: DataSetIteratorTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testLfwModel() throws Exception {
        final int numRows = 28;
        final int numColumns = 28;
        int numChannels = 3;
        int outputNum = LFWLoader.NUM_LABELS;
        int numSamples = LFWLoader.NUM_IMAGES;
        int batchSize = 2;
        int seed = 123;
        int listenerFreq = 1;

        LFWDataSetIterator lfw = new LFWDataSetIterator(batchSize, numSamples,
                        new int[] {numRows, numColumns, numChannels}, outputNum, false, true, 1.0, new Random(seed));

        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                        .gradientNormalization(GradientNormalization.RenormalizeL2PerLayer)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).list()
                        .layer(0, new ConvolutionLayer.Builder(5, 5).nIn(numChannels).nOut(6)
                                        .weightInit(WeightInit.XAVIER).activation(Activation.RELU).build())
                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                        .stride(1, 1).build())
                        .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                        .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                        .build())
                        .setInputType(InputType.convolutionalFlat(numRows, numColumns, numChannels))
                        ;

        MultiLayerNetwork model = new MultiLayerNetwork(builder.build());
        model.init();

        model.setListeners(new ScoreIterationListener(listenerFreq));

        model.fit(lfw.next());

        DataSet dataTest = lfw.next();
        INDArray output = model.output(dataTest.getFeatures());
        Evaluation eval = new Evaluation(outputNum);
        eval.eval(dataTest.getLabels(), output);
//        System.out.println(eval.stats());
    }
 
Example #27
Source File: TestUpdaters.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testDivisionByMinibatch1(){
    //No batch norm - should be single INDArray equal to flattened gradient view

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    net.fit(Nd4j.create(1,10), Nd4j.create(1,10));

    BaseMultiLayerUpdater u = (BaseMultiLayerUpdater) net.getUpdater();
    List<INDArray> l = u.getGradientsForMinibatchDivision();
    assertNotNull(l);
    assertEquals(1, l.size());

    INDArray arr = l.get(0);
    assertEquals(3 * (10 * 10 + 10), arr.length());
    assertEquals(net.getFlattenedGradients(), arr);
}
 
Example #28
Source File: GravesLSTMTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testGateActivationFnsSanityCheck() {
    for (String gateAfn : new String[] {"sigmoid", "hardsigmoid"}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                        .seed(12345).list()
                        .layer(0, new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder()
                                        .gateActivationFunction(gateAfn).activation(Activation.TANH).nIn(2).nOut(2)
                                        .build())
                        .layer(1, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder()
                                        .lossFunction(LossFunctions.LossFunction.MSE).nIn(2).nOut(2)
                                        .activation(Activation.TANH).build())
                        .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        assertEquals(gateAfn, ((org.deeplearning4j.nn.conf.layers.GravesLSTM) net.getLayer(0).conf().getLayer())
                        .getGateActivationFn().toString());

        INDArray in = Nd4j.rand(new int[] {3, 2, 5});
        INDArray labels = Nd4j.rand(new int[] {3, 2, 5});

        net.fit(in, labels);
    }
}
 
Example #29
Source File: TestEarlyStoppingCompGraph.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoImprovementNEpochsTermination() {
    //Idea: terminate training if score (test set loss) does not improve for 5 consecutive epochs
    //Simulate this by setting LR = 0.0

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(0.0)).weightInit(WeightInit.XAVIER).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new OutputLayer.Builder().nIn(4).nOut(3)
                            .activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build(), "in")
                    .setOutputs("0").build();
    ComputationGraph net = new ComputationGraph(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);

    EarlyStoppingModelSaver<ComputationGraph> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<ComputationGraph> esConf = new EarlyStoppingConfiguration.Builder<ComputationGraph>()
                    .epochTerminationConditions(new MaxEpochsTerminationCondition(100),
                                    new ScoreImprovementEpochTerminationCondition(5))
                    .iterationTerminationConditions(new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                    new MaxScoreIterationTerminationCondition(50)) //Initial score is ~8
                    .scoreCalculator(new DataSetLossCalculatorCG(irisIter, true)).modelSaver(saver).build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingGraphTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    //Expect no score change due to 0 LR -> terminate after 6 total epochs
    assertEquals(6, result.getTotalEpochs());
    assertEquals(0, result.getBestModelEpoch());
    assertEquals(EarlyStoppingResult.TerminationReason.EpochTerminationCondition, result.getTerminationReason());
    String expDetails = new ScoreImprovementEpochTerminationCondition(5).toString();
    assertEquals(expDetails, result.getTerminationDetails());
}
 
Example #30
Source File: TestComputationGraphNetwork.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testDisconnectedVertex(){

    for(boolean allowDisconnected : new boolean[]{false, true}) {
        try {
            ComputationGraphConfiguration.GraphBuilder b = new NeuralNetConfiguration.Builder()
                    .graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new DenseLayer.Builder().activation(Activation.SIGMOID).nOut(8).build(), "in")
                    .addLayer("1", new DenseLayer.Builder().activation(Activation.SIGMOID).nOut(8).build(), "in") //Disconnected
                    .addLayer("O", new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nOut(10).build(), "0")
                    .setOutputs("O")
                    .setInputTypes(InputType.feedForward(8));

            if(allowDisconnected){
                b.allowDisconnected(true).build();  //No exception
            } else {
                b.build();  //Expect exception here
                fail("Expected exception for disconnected vertex");
            }


        } catch (Exception e) {
            log.error("",e);
            if(allowDisconnected){
                fail("No exception expected");
            } else {
                String msg = e.getMessage().toLowerCase();
                assertTrue(msg.contains("disconnected"));
            }
        }
    }
}