Java Code Examples for org.nd4j.linalg.lossfunctions.LossFunctions

The following examples show how to use org.nd4j.linalg.lossfunctions.LossFunctions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: deeplearning4j   Source File: LocalResponseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiCNNLayer() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new LocalResponseNormalization.Builder().build()).layer(2,
                                    new DenseLayer.Builder()
                                                    .nOut(2).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10)
                                    .build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    network.fit(next);
}
 
Example 2
Source Project: deeplearning4j   Source File: ComputationGraphTestRNN.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidTPBTT() {
    int nIn = 8;
    int nOut = 25;
    int nHiddenUnits = 17;

    try {
        new NeuralNetConfiguration.Builder()
                .graphBuilder()
                .addInputs("in")
                .layer("0", new org.deeplearning4j.nn.conf.layers.LSTM.Builder().nIn(nIn).nOut(nHiddenUnits).build(), "in")
                .layer("1", new GlobalPoolingLayer(), "0")
                .layer("2", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(nHiddenUnits)
                        .nOut(nOut)
                        .activation(Activation.TANH).build(), "1")
                .setOutputs("2")
                .backpropType(BackpropType.TruncatedBPTT)
                .build();
        fail("Exception expected");
    } catch (IllegalStateException e){
        log.error("",e);
        assertTrue(e.getMessage().contains("TBPTT") && e.getMessage().contains("validateTbpttConfig"));
    }
}
 
Example 3
Source Project: deeplearning4j   Source File: ConvolutionLayerSetupTest.java    License: Apache License 2.0 6 votes vote down vote up
public MultiLayerConfiguration.Builder incompleteLRN() {
    MultiLayerConfiguration.Builder builder =
                    new NeuralNetConfiguration.Builder().seed(3)
                                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(2, new LocalResponseNormalization.Builder().build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                            LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2)
                                            .activation(Activation.SOFTMAX).build());
    return builder;
}
 
Example 4
Source Project: deeplearning4j   Source File: MultiLayerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPredict() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .weightInit(WeightInit.XAVIER).seed(12345L).list()
                    .layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build())
                    .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(50).nOut(10).build())
                    .setInputType(InputType.convolutional(28, 28, 1)).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator ds = new MnistDataSetIterator(10, 10);
    net.fit(ds);

    DataSetIterator testDs = new MnistDataSetIterator(1, 1);
    DataSet testData = testDs.next();
    testData.setLabelNames(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"));
    String actualLables = testData.getLabelName(0);
    List<String> prediction = net.predict(testData);
    assertTrue(actualLables != null);
    assertTrue(prediction.get(0) != null);
}
 
Example 5
Source Project: deeplearning4j   Source File: RNNTestCases.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object getConfiguration() throws Exception {
    return new NeuralNetConfiguration.Builder()
            .dataType(DataType.FLOAT)
            .seed(12345)
            .updater(new Adam(5e-2))
            .l1(1e-3).l2(1e-3)
            .list()
            .layer(0, new LSTM.Builder().activation(Activation.TANH).nOut(10).build())
            .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build())
            .layer(new OutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.recurrent(1))
            .build();
}
 
Example 6
Source Project: deeplearning4j   Source File: TestBasic.java    License: Apache License 2.0 6 votes vote down vote up
private static MultiLayerSpace getMultiLayerSpaceMnist() {
    return new MultiLayerSpace.Builder()
            .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2)))
            .l2(new ContinuousParameterSpace(0.0001, 0.05))
            .addLayer(
                    new ConvolutionLayerSpace.Builder().nIn(1)
                            .nOut(new IntegerParameterSpace(5, 30))
                            .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3},
                                    new int[]{4, 4}, new int[]{5, 5}))
                            .stride(new DiscreteParameterSpace<>(new int[]{1, 1},
                                    new int[]{2, 2}))
                            .activation(new DiscreteParameterSpace<>(Activation.RELU,
                                    Activation.SOFTPLUS, Activation.LEAKYRELU))
                            .build())
            .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128))
                    .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH))
                    .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers
            .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1))
            .build();
}
 
Example 7
Source Project: deeplearning4j   Source File: DenseTest.java    License: Apache License 2.0 6 votes vote down vote up
private static MultiLayerNetwork getDenseMLNConfig(boolean backprop, boolean pretrain) {
    int numInputs = 4;
    int outputNum = 3;
    long seed = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
                    .updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    return model;

}
 
Example 8
Source Project: deeplearning4j   Source File: TestErrors.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 20000L)
public void testAllInvalidDataConfigMismatch() throws Exception {
    //Valid config - but mismatched with provided data

    File f = temp.newFolder();
    MultiLayerSpace mls = new MultiLayerSpace.Builder()
            .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(10)    //INVALID: nOut of 0
                    .activation(Activation.TANH)
                    .build())
            .addLayer(new OutputLayerSpace.Builder().nIn(10).nOut(3).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();

    CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls);

    OptimizationConfiguration configuration = new OptimizationConfiguration.Builder()
            .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3))
            .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true))
            .terminationConditions(
                    new MaxCandidatesCondition(5))
            .build();

    IOptimizationRunner runner = new LocalOptimizationRunner(configuration);
    runner.execute();
}
 
Example 9
Source Project: deeplearning4j   Source File: ScoringModelTest.java    License: Apache License 2.0 6 votes vote down vote up
protected Model buildMultiLayerNetworkModel(int numFeatures) throws Exception {

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder().nIn(numFeatures).nOut(1).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[numFeatures+1];
    float base = 1f;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example 10
Source Project: deeplearning4j   Source File: TestLrChanges.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testChangeLSGD() {
    //Simple test for no updater nets
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .activation(Activation.TANH)
            .seed(12345)
            .updater(new Sgd(0.1))
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new OutputLayer.Builder().nIn(10).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    net.setLearningRate(1.0);
    net.setLearningRate(1, 0.5);
    assertEquals(1.0, net.getLearningRate(0), 0.0);
    assertEquals(0.5, net.getLearningRate(1), 0.0);


    ComputationGraph cg = net.toComputationGraph();
    cg.setLearningRate(2.0);
    cg.setLearningRate("1", 2.5);
    assertEquals(2.0, cg.getLearningRate("0"), 0.0);
    assertEquals(2.5, cg.getLearningRate("1"), 0.0);

}
 
Example 11
Source Project: deeplearning4j   Source File: ActivationLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testActivationInheritance() {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.RATIONALTANH)
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new ActivationLayer())
            .layer(new ActivationLayer.Builder().build())
            .layer(new ActivationLayer.Builder().activation(Activation.ELU).build())
            .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    assertNotNull(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn());

    assertTrue(((DenseLayer)network.getLayer(0).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(2).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(3).conf().getLayer()).getActivationFn() instanceof ActivationELU);
    assertTrue(((OutputLayer)network.getLayer(4).conf().getLayer()).getActivationFn() instanceof ActivationSoftmax);
}
 
Example 12
Source Project: deeplearning4j   Source File: TestFailureListener.java    License: Apache License 2.0 5 votes vote down vote up
@Ignore
@Test
public void testFailureRandom_AND() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1e-4))
            .list()
            .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    String hostname = InetAddress.getLocalHost().getHostName();
    assertNotNull(hostname);
    assertFalse(hostname.isEmpty());

    net.setListeners(new FailureTestingListener(
            FailureTestingListener.FailureMode.ILLEGAL_STATE,
            new FailureTestingListener.And(
                    new FailureTestingListener.HostNameTrigger(hostname),
                    new FailureTestingListener.RandomProb(FailureTestingListener.CallType.ANY, 0.05))
    ));

    DataSetIterator iter = new IrisDataSetIterator(5,150);

    net.fit(iter);
}
 
Example 13
Source Project: deeplearning4j   Source File: EvalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEvalSplitting2(){
    List<List<Writable>> seqFeatures = new ArrayList<>();
    List<Writable> step = Arrays.<Writable>asList(new FloatWritable(0), new FloatWritable(0), new FloatWritable(0));
    for( int i=0; i<30; i++ ){
        seqFeatures.add(step);
    }
    List<List<Writable>> seqLabels = Collections.singletonList(Collections.<Writable>singletonList(new FloatWritable(0)));

    SequenceRecordReader fsr = new CollectionSequenceRecordReader(Collections.singletonList(seqFeatures));
    SequenceRecordReader lsr = new CollectionSequenceRecordReader(Collections.singletonList(seqLabels));


    DataSetIterator testData = new SequenceRecordReaderDataSetIterator(fsr, lsr, 1, -1, true,
            SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123)
            .list()
            .layer(0, new LSTM.Builder().activation(Activation.TANH).nIn(3).nOut(3).build())
            .layer(1, new RnnOutputLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.XENT)
                    .nIn(3).nOut(1).build())
            .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(10).tBPTTBackwardLength(10)
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    net.evaluate(testData);
}
 
Example 14
@Test
public void testConvnetJson() {
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l1(1e-1).l2(2e-4).weightNoise(new DropConnect(0.5)).miniBatch(true)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                    .build())

                    .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    MultiLayerConfiguration conf = builder.build();
    String json = conf.toJson();
    MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 
Example 15
Source Project: deeplearning4j   Source File: ScoringModelTest.java    License: Apache License 2.0 5 votes vote down vote up
protected Model buildComputationGraphModel(int numFeatures) throws Exception {

    final ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
        .graphBuilder()
        .addInputs("inputLayer")
        .addLayer("outputLayer",
          new OutputLayer.Builder().nIn(numFeatures).nOut(1).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build(),
          "inputLayer")
        .setOutputs("outputLayer")
        .build();

    final ComputationGraph model = new ComputationGraph(conf);
    model.init();

    final float[] floats = new float[numFeatures+1];
    float base = 1f;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example 16
Source Project: deeplearning4j   Source File: TransferLearningMLNTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTransferLearningSubsequent() {
    final INDArray input = Nd4j.create(6,6,6,6);
    final MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
            .weightInit(new ConstantDistribution(666))
            .list()
            .setInputType(InputType.inferInputTypes(input)[0])
            .layer(new Convolution2D.Builder(3, 3).nOut(10).build())
            .layer(new Convolution2D.Builder(1, 1).nOut(3).build())
            .layer(new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE)
                    .build()).build());
    net.init();

    MultiLayerNetwork newGraph = new TransferLearning
            .Builder(net)
            .fineTuneConfiguration(new FineTuneConfiguration.Builder().build())
            .nOutReplace(0, 7, new ConstantDistribution(333))
            .nOutReplace(1, 3, new ConstantDistribution(111))
            .removeLayersFromOutput(1)
            .addLayer(new OutputLayer.Builder()
                    .nIn(48).nOut(2)
                    .lossFunction(LossFunctions.LossFunction.MSE)
                    .build())
            .setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(4,4,3))
            .build();
    newGraph.init();

    assertEquals("Incorrect # inputs", 7, newGraph.layerInputSize(1));

    newGraph.output(input);
}
 
Example 17
Source Project: dl4j-tutorials   Source File: MultiRegression.java    License: MIT License 5 votes vote down vote up
public static void main(String[] args){

        //Generate the training data
        DataSetIterator iterator = getTrainingData(batchSize,rng);

        //Create the network
        int numInput = 2;
        int numOutputs = 1;
        MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
                .seed(seed)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .weightInit(WeightInit.XAVIER)
                .updater(new Sgd(learningRate))
                .list()
                .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE)
                        .activation(Activation.IDENTITY)
                        .nIn(numInput).nOut(numOutputs).build())
                .pretrain(false).backprop(true).build()
        );
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        for( int i=0; i<nEpochs; i++ ){
            iterator.reset();
            net.fit(iterator);
        }

        final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 });
        INDArray out = net.output(input, false);
        System.out.println(out);

    }
 
Example 18
@Test
public void testSerialization() {

    final MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new AdaGrad(0.1))
                    .l2(0.001)
                    .seed(12345).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder()
                                    .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .nIn(2).nOut(2).build())
                    .build();


    final String json1 = conf1.toJson();

    final MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json1);

    final String json2 = conf1.toJson();


    TestCase.assertEquals(json1, json2);
}
 
Example 19
Source Project: dl4j-tutorials   Source File: Gan4Exemple.java    License: MIT License 5 votes vote down vote up
private static void discInit() throws IOException {
    discUpdater = new RmsProp(drate);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(1234)
            .updater(new RmsProp(1e-3))
            .weightInit(WeightInit.XAVIER)
            .list()
            .layer(new DenseLayer.Builder()
                    .nIn(width * height)
                    .nOut(1024)
                    .activation(Activation.RELU)
                    .build())
            .layer(new DenseLayer.Builder()
                    .nIn(1024)
                    .nOut(512)
                    .activation(Activation.RELU)
                    .build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nIn(512)
                    .nOut(2)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.feedForward(width * height))
            .backprop(true).pretrain(false).build();

    discNet = new MultiLayerNetwork(conf);
    discNet.init();
    discNet.setListeners(new ScoreIterationListener(10));

}
 
Example 20
Source Project: deeplearning4j   Source File: ManualTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCNNActivationsFrozen() throws Exception {

    int nChannels = 1;
    int outputNum = 10;
    int batchSize = 64;
    int nEpochs = 10;
    int seed = 123;

    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);

    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new FrozenLayer(new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()))
            .layer(1, new FrozenLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build()))
            .layer(2, new FrozenLayer(new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()))
            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    log.info("Train model....");
    model.setListeners(new ConvolutionalIterationListener(1));

    for (int i = 0; i < nEpochs; i++) {
        model.fit(mnistTrain);
    }
}
 
Example 21
Source Project: deeplearning4j   Source File: BatchNormalizationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchNormRecurrentCnn1d() {
    //Simple sanity check on CNN1D and RNN layers

    for (boolean rnn : new boolean[]{true, false}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .weightInit(WeightInit.XAVIER)
                .convolutionMode(ConvolutionMode.Same)
                .list()
                .layer(rnn ? new LSTM.Builder().nOut(3).build() :
                        new Convolution1DLayer.Builder().kernelSize(3).stride(1).nOut(3).build())
                .layer(new BatchNormalization())
                .layer(new RnnOutputLayer.Builder().nOut(3).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build())
                .setInputType(InputType.recurrent(3))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray in = Nd4j.rand(new int[]{1, 3, 5});
        INDArray label = Nd4j.rand(new int[]{1, 3, 5});

        INDArray out = net.output(in);
        assertArrayEquals(new long[]{1, 3, 5}, out.shape());

        net.fit(in, label);
        log.info("OK: {}", (rnn ? "rnn" : "cnn1d"));
    }
}
 
Example 22
private static MultiLayerConfiguration getConf(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build())
                    .build();
    return conf;
}
 
Example 23
private static Model buildModel() throws Exception {

    final int numInputs = 3;
    final int numOutputs = 2;

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 };
    // positive weight for first output, negative weight for second output, no biases
    assertEquals((numInputs+1)*numOutputs, floats.length);

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example 24
Source Project: deeplearning4j   Source File: MultiLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
                    .list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build();


    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getLayerWiseConfigurations().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getLayerWiseConfigurations().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(bais, true);
    assertEquals(7, net.getLayerWiseConfigurations().getIterationCount());
}
 
Example 25
Source Project: deeplearning4j   Source File: BNGradientCheckTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
    public void testGradient2dFixedGammaBeta() {
        DataNormalization scaler = new NormalizerMinMaxScaler();
        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        scaler.fit(iter);
        iter.setPreProcessor(scaler);
        DataSet ds = iter.next();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        for (boolean useLogStd : new boolean[]{true, false}) {
            MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 1)).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build())
                    .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).nOut(3)
                            .build())
                    .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nIn(3).nOut(3).build());

            MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
            mln.init();

//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());

            //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
            //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
            //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
            Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input)
                    .labels(labels).excludeParams(excludeParams));

            assertTrue(gradOK);
            TestUtils.testModelSerialization(mln);
        }
    }
 
Example 26
Source Project: deeplearning4j   Source File: ComputationGraphTestRNN.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTBPTTLongerThanTS() {
    int tbpttLength = 100;
    int timeSeriesLength = 20;
    int miniBatchSize = 7;
    int nIn = 5;
    int nOut = 4;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
                                    .activation(Activation.TANH)
                                    .dist(new NormalDistribution(0, 0.5)).build(), "in")
                    .addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
                                    .activation(Activation.TANH)
                                    .dist(new NormalDistribution(0,
                                                    0.5))
                                    .build(), "0")
                    .addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .nIn(8).nOut(nOut)
                                    .activation(Activation.SOFTMAX)
                                    .dist(new NormalDistribution(0, 0.5)).build(), "1")
                    .setOutputs("out").backpropType(BackpropType.TruncatedBPTT)
                    .tBPTTBackwardLength(tbpttLength).tBPTTForwardLength(tbpttLength).build();

    Nd4j.getRandom().setSeed(12345);
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength});
    INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, timeSeriesLength});

    INDArray initialParams = graph.params().dup();
    graph.fit(new INDArray[] {inputLong}, new INDArray[] {labelsLong});
    INDArray afterParams = graph.params();

    assertNotEquals(initialParams, afterParams);
}
 
Example 27
Source Project: deeplearning4j   Source File: TestVAE.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJsonYaml() {

    MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.IDENTITY))
                                    .nIn(3).nOut(4).encoderLayerSizes(5).decoderLayerSizes(6).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
                                    .nIn(7).nOut(8).encoderLayerSizes(9).decoderLayerSizes(10).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new BernoulliReconstructionDistribution()).nIn(11)
                                    .nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new ExponentialReconstructionDistribution(Activation.TANH))
                                    .nIn(11).nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(4, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .lossFunction(new ActivationTanH(), LossFunctions.LossFunction.MSE).nIn(11)
                                    .nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(5, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new CompositeReconstructionDistribution.Builder()
                                                    .addDistribution(5, new GaussianReconstructionDistribution())
                                                    .addDistribution(5,
                                                                    new GaussianReconstructionDistribution(Activation.TANH))
                                                    .addDistribution(5, new BernoulliReconstructionDistribution())
                                                    .build())
                                    .nIn(15).nOut(16).encoderLayerSizes(17).decoderLayerSizes(18).build())
                    .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(18)
                                    .nOut(19).activation(new ActivationTanH()).build())
                    .build();

    String asJson = config.toJson();
    String asYaml = config.toYaml();

    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson);
    MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(asYaml);

    assertEquals(config, fromJson);
    assertEquals(config, fromYaml);
}
 
Example 28
Source Project: deeplearning4j   Source File: EvalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluativeListenerSimple(){
    //Sanity check: https://github.com/deeplearning4j/deeplearning4j/issues/5351

    // Network config
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()

            .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42)
            .updater(new Sgd(1e-6)).list()
            .layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build())
            .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                    LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.SOFTMAX).build())
            .build();

    // Instantiate model
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    // Train-test split
    DataSetIterator iter = new IrisDataSetIterator(30, 150);
    DataSetIterator iterTest = new IrisDataSetIterator(30, 150);

    net.setListeners(new EvaluativeListener(iterTest, 3));

    for( int i=0; i<3; i++ ){
        net.fit(iter);
    }
}
 
Example 29
Source Project: deeplearning4j   Source File: MultiLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCompareLayerMethods(){
    //Simple test: compare .layer(int, Layer) and .layer(Layer) are identical

    MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list()
            .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(2, new LSTM.Builder().nIn(2).nOut(2).build())
            .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3)
                    .build())
            .build();

    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123).list()
            .layer(new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(new LSTM.Builder().nIn(2).nOut(2).build())
            .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3)
                    .build())
            .build();

    assertEquals(conf1, conf2);
}
 
Example 30
public static void main(String[] args) throws IOException, InterruptedException {
    if(FEATURE_DIR.equals("{PATH-TO-PHYSIONET-FEATURES}") || LABEL_DIR.equals("{PATH-TO-PHYSIONET-LABELS")){
        System.out.println("Please provide proper directory path in place of: PATH-TO-PHYSIONET-FEATURES && PATH-TO-PHYSIONET-LABELS");
        throw new FileNotFoundException();
    }
    SequenceRecordReader trainFeaturesReader = new CSVSequenceRecordReader(1, ",");
    trainFeaturesReader.initialize(new NumberedFileInputSplit(FEATURE_DIR+"/%d.csv",0,3199));
    SequenceRecordReader trainLabelsReader = new CSVSequenceRecordReader();
    trainLabelsReader.initialize(new NumberedFileInputSplit(LABEL_DIR+"/%d.csv",0,3199));
    DataSetIterator trainDataSetIterator = new SequenceRecordReaderDataSetIterator(trainFeaturesReader,trainLabelsReader,100,2,false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    SequenceRecordReader testFeaturesReader = new CSVSequenceRecordReader(1, ",");
    testFeaturesReader.initialize(new NumberedFileInputSplit(FEATURE_DIR+"/%d.csv",3200,3999));
    SequenceRecordReader testLabelsReader = new CSVSequenceRecordReader();
    testLabelsReader.initialize(new NumberedFileInputSplit(LABEL_DIR+"/%d.csv",3200,3999));
    DataSetIterator testDataSetIterator = new SequenceRecordReaderDataSetIterator(testFeaturesReader,testLabelsReader,100,2,false, SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    ComputationGraphConfiguration configuration = new NeuralNetConfiguration.Builder()
                                                    .seed(RANDOM_SEED)
                                                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                                                    .weightInit(WeightInit.XAVIER)
                                                    .updater(new Adam())
                                                    .dropOut(0.9)
                                                    .graphBuilder()
                                                    .addInputs("trainFeatures")
                                                    .setOutputs("predictMortality")
                                                    .addLayer("L1", new LSTM.Builder()
                                                                                   .nIn(86)
                                                                                    .nOut(200)
                                                                                    .forgetGateBiasInit(1)
                                                                                    .activation(Activation.TANH)
                                                                                    .build(),"trainFeatures")
                                                    .addLayer("predictMortality", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                                                                        .activation(Activation.SOFTMAX)
                                                                                        .nIn(200).nOut(2).build(),"L1")
                                                    .build();

    ComputationGraph model = new ComputationGraph(configuration);

    for(int i=0;i<1;i++){
       model.fit(trainDataSetIterator);
       trainDataSetIterator.reset();
    }
    ROC evaluation = new ROC(100);
    while (testDataSetIterator.hasNext()) {
        DataSet batch = testDataSetIterator.next();
        INDArray[] output = model.output(batch.getFeatures());
        evaluation.evalTimeSeries(batch.getLabels(), output[0]);
    }
    
    System.out.println(evaluation.calculateAUC());
    System.out.println(evaluation.stats());
}