org.nd4j.linalg.lossfunctions.LossFunctions Java Examples

The following examples show how to use org.nd4j.linalg.lossfunctions.LossFunctions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConvolutionLayerSetupTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public MultiLayerConfiguration.Builder incompleteLRN() {
    MultiLayerConfiguration.Builder builder =
                    new NeuralNetConfiguration.Builder().seed(3)
                                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                                    .layer(0, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(1, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(2, new LocalResponseNormalization.Builder().build())
                                    .layer(3, new org.deeplearning4j.nn.conf.layers.ConvolutionLayer.Builder(
                                                    new int[] {5, 5}).nOut(6).build())
                                    .layer(4, new org.deeplearning4j.nn.conf.layers.SubsamplingLayer.Builder(
                                                    new int[] {2, 2}).build())
                                    .layer(5, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                            LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD).nOut(2)
                                            .activation(Activation.SOFTMAX).build());
    return builder;
}
 
Example #2
Source File: ScoringModelTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
protected Model buildMultiLayerNetworkModel(int numFeatures) throws Exception {

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder().nIn(numFeatures).nOut(1).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[numFeatures+1];
    float base = 1f;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example #3
Source File: LocalResponseTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiCNNLayer() throws Exception {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(123).list()
                    .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new LocalResponseNormalization.Builder().build()).layer(2,
                                    new DenseLayer.Builder()
                                                    .nOut(2).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(10)
                                    .build())
                    .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    DataSetIterator iter = new MnistDataSetIterator(2, 2);
    DataSet next = iter.next();

    network.fit(next);
}
 
Example #4
Source File: MultiLayerTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testPredict() throws Exception {

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .weightInit(WeightInit.XAVIER).seed(12345L).list()
                    .layer(0, new DenseLayer.Builder().nIn(784).nOut(50).activation(Activation.RELU).build())
                    .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .activation(Activation.SOFTMAX).nIn(50).nOut(10).build())
                    .setInputType(InputType.convolutional(28, 28, 1)).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator ds = new MnistDataSetIterator(10, 10);
    net.fit(ds);

    DataSetIterator testDs = new MnistDataSetIterator(1, 1);
    DataSet testData = testDs.next();
    testData.setLabelNames(Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"));
    String actualLables = testData.getLabelName(0);
    List<String> prediction = net.predict(testData);
    assertTrue(actualLables != null);
    assertTrue(prediction.get(0) != null);
}
 
Example #5
Source File: RNNTestCases.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
public Object getConfiguration() throws Exception {
    return new NeuralNetConfiguration.Builder()
            .dataType(DataType.FLOAT)
            .seed(12345)
            .updater(new Adam(5e-2))
            .l1(1e-3).l2(1e-3)
            .list()
            .layer(0, new LSTM.Builder().activation(Activation.TANH).nOut(10).build())
            .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.AVG).build())
            .layer(new OutputLayer.Builder().nOut(6)
                    .lossFunction(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.recurrent(1))
            .build();
}
 
Example #6
Source File: TestBasic.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private static MultiLayerSpace getMultiLayerSpaceMnist() {
    return new MultiLayerSpace.Builder()
            .updater(new SgdSpace(new ContinuousParameterSpace(0.0001, 0.2)))
            .l2(new ContinuousParameterSpace(0.0001, 0.05))
            .addLayer(
                    new ConvolutionLayerSpace.Builder().nIn(1)
                            .nOut(new IntegerParameterSpace(5, 30))
                            .kernelSize(new DiscreteParameterSpace<>(new int[]{3, 3},
                                    new int[]{4, 4}, new int[]{5, 5}))
                            .stride(new DiscreteParameterSpace<>(new int[]{1, 1},
                                    new int[]{2, 2}))
                            .activation(new DiscreteParameterSpace<>(Activation.RELU,
                                    Activation.SOFTPLUS, Activation.LEAKYRELU))
                            .build())
            .addLayer(new DenseLayerSpace.Builder().nOut(new IntegerParameterSpace(32, 128))
                    .activation(new DiscreteParameterSpace<>(Activation.RELU, Activation.TANH))
                    .build(), new IntegerParameterSpace(0, 1), true) //0 to 1 layers
            .addLayer(new OutputLayerSpace.Builder().nOut(10).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1))
            .build();
}
 
Example #7
Source File: DenseTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private static MultiLayerNetwork getDenseMLNConfig(boolean backprop, boolean pretrain) {
    int numInputs = 4;
    int outputNum = 3;
    long seed = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
                    .updater(new Sgd(1e-3)).l1(0.3).l2(1e-3).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(numInputs).nOut(3)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.DenseLayer.Builder().nIn(3).nOut(2)
                                    .activation(Activation.TANH).weightInit(WeightInit.XAVIER).build())
                    .layer(2, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .weightInit(WeightInit.XAVIER).nIn(2).nOut(outputNum).activation(Activation.SOFTMAX).build())
                    .build();

    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();
    return model;

}
 
Example #8
Source File: TestErrors.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 20000L)
public void testAllInvalidDataConfigMismatch() throws Exception {
    //Valid config - but mismatched with provided data

    File f = temp.newFolder();
    MultiLayerSpace mls = new MultiLayerSpace.Builder()
            .addLayer(new DenseLayerSpace.Builder().nIn(4).nOut(10)    //INVALID: nOut of 0
                    .activation(Activation.TANH)
                    .build())
            .addLayer(new OutputLayerSpace.Builder().nIn(10).nOut(3).activation(Activation.SOFTMAX)
                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();

    CandidateGenerator candidateGenerator = new RandomSearchGenerator(mls);

    OptimizationConfiguration configuration = new OptimizationConfiguration.Builder()
            .candidateGenerator(candidateGenerator).dataProvider(new TestDataProviderMnist(32, 3))
            .modelSaver(new FileModelSaver(f)).scoreFunction(new TestSetLossScoreFunction(true))
            .terminationConditions(
                    new MaxCandidatesCondition(5))
            .build();

    IOptimizationRunner runner = new LocalOptimizationRunner(configuration);
    runner.execute();
}
 
Example #9
Source File: ComputationGraphTestRNN.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidTPBTT() {
    int nIn = 8;
    int nOut = 25;
    int nHiddenUnits = 17;

    try {
        new NeuralNetConfiguration.Builder()
                .graphBuilder()
                .addInputs("in")
                .layer("0", new org.deeplearning4j.nn.conf.layers.LSTM.Builder().nIn(nIn).nOut(nHiddenUnits).build(), "in")
                .layer("1", new GlobalPoolingLayer(), "0")
                .layer("2", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(nHiddenUnits)
                        .nOut(nOut)
                        .activation(Activation.TANH).build(), "1")
                .setOutputs("2")
                .backpropType(BackpropType.TruncatedBPTT)
                .build();
        fail("Exception expected");
    } catch (IllegalStateException e){
        log.error("",e);
        assertTrue(e.getMessage().contains("TBPTT") && e.getMessage().contains("validateTbpttConfig"));
    }
}
 
Example #10
Source File: MultiRegression.java    From dl4j-tutorials with MIT License 5 votes vote down vote up
public static void main(String[] args){

        //Generate the training data
        DataSetIterator iterator = getTrainingData(batchSize,rng);

        //Create the network
        int numInput = 2;
        int numOutputs = 1;
        MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
                .seed(seed)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                .weightInit(WeightInit.XAVIER)
                .updater(new Sgd(learningRate))
                .list()
                .layer(0, new OutputLayer.Builder(LossFunctions.LossFunction.MSE)
                        .activation(Activation.IDENTITY)
                        .nIn(numInput).nOut(numOutputs).build())
                .pretrain(false).backprop(true).build()
        );
        net.init();
        net.setListeners(new ScoreIterationListener(1));

        for( int i=0; i<nEpochs; i++ ){
            iterator.reset();
            net.fit(iterator);
        }

        final INDArray input = Nd4j.create(new double[] { 0.111111, 0.3333333333333 }, new int[] { 1, 2 });
        INDArray out = net.output(input, false);
        System.out.println(out);

    }
 
Example #11
Source File: Gan4Exemple.java    From dl4j-tutorials with MIT License 5 votes vote down vote up
private static void discInit() throws IOException {
    discUpdater = new RmsProp(drate);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(1234)
            .updater(new RmsProp(1e-3))
            .weightInit(WeightInit.XAVIER)
            .list()
            .layer(new DenseLayer.Builder()
                    .nIn(width * height)
                    .nOut(1024)
                    .activation(Activation.RELU)
                    .build())
            .layer(new DenseLayer.Builder()
                    .nIn(1024)
                    .nOut(512)
                    .activation(Activation.RELU)
                    .build())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nIn(512)
                    .nOut(2)
                    .activation(Activation.SOFTMAX)
                    .build())
            .setInputType(InputType.feedForward(width * height))
            .backprop(true).pretrain(false).build();

    discNet = new MultiLayerNetwork(conf);
    discNet.init();
    discNet.setListeners(new ScoreIterationListener(10));

}
 
Example #12
Source File: GravesBidirectionalLSTMTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() {

    final MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new AdaGrad(0.1))
                    .l2(0.001)
                    .seed(12345).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.GravesBidirectionalLSTM.Builder()
                                    .activation(Activation.TANH).nIn(2).nOut(2)
                                    .dist(new UniformDistribution(-0.05, 0.05)).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.RnnOutputLayer.Builder()
                                    .activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT)
                                    .nIn(2).nOut(2).build())
                    .build();


    final String json1 = conf1.toJson();

    final MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json1);

    final String json2 = conf1.toJson();


    TestCase.assertEquals(json1, json2);
}
 
Example #13
Source File: MultiLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCompareLayerMethods(){
    //Simple test: compare .layer(int, Layer) and .layer(Layer) are identical

    MultiLayerConfiguration conf1 = new NeuralNetConfiguration.Builder().seed(123).list()
            .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(1, new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(2, new LSTM.Builder().nIn(2).nOut(2).build())
            .layer(3, new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3)
                    .build())
            .build();

    MultiLayerConfiguration conf2 = new NeuralNetConfiguration.Builder().seed(123).list()
            .layer(new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(new DenseLayer.Builder().nIn(3).nOut(2).weightInit(WeightInit.XAVIER)
                    .activation(Activation.TANH).build())
            .layer(new LSTM.Builder().nIn(2).nOut(2).build())
            .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nIn(2).nOut(3)
                    .build())
            .build();

    assertEquals(conf1, conf2);
}
 
Example #14
Source File: TransferLearningMLNTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testTransferLearningSubsequent() {
    final INDArray input = Nd4j.create(6,6,6,6);
    final MultiLayerNetwork net = new MultiLayerNetwork(new NeuralNetConfiguration.Builder()
            .weightInit(new ConstantDistribution(666))
            .list()
            .setInputType(InputType.inferInputTypes(input)[0])
            .layer(new Convolution2D.Builder(3, 3).nOut(10).build())
            .layer(new Convolution2D.Builder(1, 1).nOut(3).build())
            .layer(new OutputLayer.Builder().nOut(2).lossFunction(LossFunctions.LossFunction.MSE)
                    .build()).build());
    net.init();

    MultiLayerNetwork newGraph = new TransferLearning
            .Builder(net)
            .fineTuneConfiguration(new FineTuneConfiguration.Builder().build())
            .nOutReplace(0, 7, new ConstantDistribution(333))
            .nOutReplace(1, 3, new ConstantDistribution(111))
            .removeLayersFromOutput(1)
            .addLayer(new OutputLayer.Builder()
                    .nIn(48).nOut(2)
                    .lossFunction(LossFunctions.LossFunction.MSE)
                    .build())
            .setInputPreProcessor(2, new CnnToFeedForwardPreProcessor(4,4,3))
            .build();
    newGraph.init();

    assertEquals("Incorrect # inputs", 7, newGraph.layerInputSize(1));

    newGraph.output(input);
}
 
Example #15
Source File: ScoringModelTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
protected Model buildComputationGraphModel(int numFeatures) throws Exception {

    final ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder()
        .graphBuilder()
        .addInputs("inputLayer")
        .addLayer("outputLayer",
          new OutputLayer.Builder().nIn(numFeatures).nOut(1).lossFunction(LossFunctions.LossFunction.MSE).activation(Activation.IDENTITY).build(),
          "inputLayer")
        .setOutputs("outputLayer")
        .build();

    final ComputationGraph model = new ComputationGraph(conf);
    model.init();

    final float[] floats = new float[numFeatures+1];
    float base = 1f;
    for (int ii=0; ii<floats.length; ++ii)
    {
      base *= 2;
      floats[ii] = base;
    }

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example #16
Source File: EvalTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvalSplitting2(){
    List<List<Writable>> seqFeatures = new ArrayList<>();
    List<Writable> step = Arrays.<Writable>asList(new FloatWritable(0), new FloatWritable(0), new FloatWritable(0));
    for( int i=0; i<30; i++ ){
        seqFeatures.add(step);
    }
    List<List<Writable>> seqLabels = Collections.singletonList(Collections.<Writable>singletonList(new FloatWritable(0)));

    SequenceRecordReader fsr = new CollectionSequenceRecordReader(Collections.singletonList(seqFeatures));
    SequenceRecordReader lsr = new CollectionSequenceRecordReader(Collections.singletonList(seqLabels));


    DataSetIterator testData = new SequenceRecordReaderDataSetIterator(fsr, lsr, 1, -1, true,
            SequenceRecordReaderDataSetIterator.AlignmentMode.ALIGN_END);

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(123)
            .list()
            .layer(0, new LSTM.Builder().activation(Activation.TANH).nIn(3).nOut(3).build())
            .layer(1, new RnnOutputLayer.Builder().activation(Activation.SIGMOID).lossFunction(LossFunctions.LossFunction.XENT)
                    .nIn(3).nOut(1).build())
            .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(10).tBPTTBackwardLength(10)
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    net.evaluate(testData);
}
 
Example #17
Source File: TestFailureListener.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Ignore
@Test
public void testFailureRandom_AND() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1e-4))
            .list()
            .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    String hostname = InetAddress.getLocalHost().getHostName();
    assertNotNull(hostname);
    assertFalse(hostname.isEmpty());

    net.setListeners(new FailureTestingListener(
            FailureTestingListener.FailureMode.ILLEGAL_STATE,
            new FailureTestingListener.And(
                    new FailureTestingListener.HostNameTrigger(hostname),
                    new FailureTestingListener.RandomProb(FailureTestingListener.CallType.ANY, 0.05))
    ));

    DataSetIterator iter = new IrisDataSetIterator(5,150);

    net.fit(iter);
}
 
Example #18
Source File: ActivationLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testActivationInheritance() {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
            .weightInit(WeightInit.XAVIER)
            .activation(Activation.RATIONALTANH)
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new ActivationLayer())
            .layer(new ActivationLayer.Builder().build())
            .layer(new ActivationLayer.Builder().activation(Activation.ELU).build())
            .layer(new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nIn(10).nOut(10).build())
            .build();

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    assertNotNull(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn());

    assertTrue(((DenseLayer)network.getLayer(0).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(1).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(2).conf().getLayer()).getActivationFn() instanceof ActivationRationalTanh);
    assertTrue(((ActivationLayer)network.getLayer(3).conf().getLayer()).getActivationFn() instanceof ActivationELU);
    assertTrue(((OutputLayer)network.getLayer(4).conf().getLayer()).getActivationFn() instanceof ActivationSoftmax);
}
 
Example #19
Source File: MultiLayerNeuralNetConfigurationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testConvnetJson() {
    final int numRows = 76;
    final int numColumns = 76;
    int nChannels = 3;
    int outputNum = 6;
    int seed = 123;

    //setup the network
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
                    .l1(1e-1).l2(2e-4).weightNoise(new DropConnect(0.5)).miniBatch(true)
                    .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list()
                    .layer(0, new ConvolutionLayer.Builder(5, 5).nOut(5).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(2, new ConvolutionLayer.Builder(3, 3).nOut(10).dropOut(0.5).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.RELU).build())
                    .layer(3, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX, new int[] {2, 2})
                                    .build())
                    .layer(4, new DenseLayer.Builder().nOut(100).activation(Activation.RELU).build())
                    .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                    .nOut(outputNum).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX)
                                    .build())

                    .setInputType(InputType.convolutional(numRows, numColumns, nChannels));

    MultiLayerConfiguration conf = builder.build();
    String json = conf.toJson();
    MultiLayerConfiguration conf2 = MultiLayerConfiguration.fromJson(json);
    assertEquals(conf, conf2);
}
 
Example #20
Source File: ManualTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCNNActivationsFrozen() throws Exception {

    int nChannels = 1;
    int outputNum = 10;
    int batchSize = 64;
    int nEpochs = 10;
    int seed = 123;

    log.info("Load data....");
    DataSetIterator mnistTrain = new MnistDataSetIterator(batchSize, true, 12345);

    log.info("Build model....");
    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(seed)
            .l2(0.0005)
            .weightInit(WeightInit.XAVIER)
            .updater(new Nesterovs(0.01, 0.9)).list()
            .layer(0, new FrozenLayer(new ConvolutionLayer.Builder(5, 5)
                    //nIn and nOut specify depth. nIn here is the nChannels and nOut is the number of filters to be applied
                    .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build()))
            .layer(1, new FrozenLayer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                    .stride(2, 2).build()))
            .layer(2, new FrozenLayer(new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build()))
            .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                    .nOut(outputNum).activation(Activation.SOFTMAX).build())
            .setInputType(InputType.convolutionalFlat(28, 28, nChannels));

    MultiLayerConfiguration conf = builder.build();
    MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    log.info("Train model....");
    model.setListeners(new ConvolutionalIterationListener(1));

    for (int i = 0; i < nEpochs; i++) {
        model.fit(mnistTrain);
    }
}
 
Example #21
Source File: BatchNormalizationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchNormRecurrentCnn1d() {
    //Simple sanity check on CNN1D and RNN layers

    for (boolean rnn : new boolean[]{true, false}) {

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .seed(12345)
                .weightInit(WeightInit.XAVIER)
                .convolutionMode(ConvolutionMode.Same)
                .list()
                .layer(rnn ? new LSTM.Builder().nOut(3).build() :
                        new Convolution1DLayer.Builder().kernelSize(3).stride(1).nOut(3).build())
                .layer(new BatchNormalization())
                .layer(new RnnOutputLayer.Builder().nOut(3).activation(Activation.TANH).lossFunction(LossFunctions.LossFunction.MSE).build())
                .setInputType(InputType.recurrent(3))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        INDArray in = Nd4j.rand(new int[]{1, 3, 5});
        INDArray label = Nd4j.rand(new int[]{1, 3, 5});

        INDArray out = net.output(in);
        assertArrayEquals(new long[]{1, 3, 5}, out.shape());

        net.fit(in, label);
        log.info("OK: {}", (rnn ? "rnn" : "cnn1d"));
    }
}
 
Example #22
Source File: TestCompareParameterAveragingSparkVsSingleMachine.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
private static MultiLayerConfiguration getConf(int seed, IUpdater updater) {
    Nd4j.getRandom().setSeed(seed);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .weightInit(WeightInit.XAVIER).updater(updater).seed(seed).list()
                    .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(1, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(10).nOut(10).build())
                    .build();
    return conf;
}
 
Example #23
Source File: ModelTupleStreamIntegrationTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
private static Model buildModel() throws Exception {

    final int numInputs = 3;
    final int numOutputs = 2;

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 };
    // positive weight for first output, negative weight for second output, no biases
    assertEquals((numInputs+1)*numOutputs, floats.length);

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example #24
Source File: MultiLayerTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testIterationCountAndPersistence() throws IOException {
    Nd4j.getRandom().setSeed(123);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(123)
                    .list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                    .activation(Activation.TANH).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                                    LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(3).nOut(3)
                                                    .build())
                    .build();


    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();

    DataSetIterator iter = new IrisDataSetIterator(50, 150);

    assertEquals(0, network.getLayerWiseConfigurations().getIterationCount());
    network.fit(iter);
    assertEquals(3, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter);
    assertEquals(6, network.getLayerWiseConfigurations().getIterationCount());
    iter.reset();
    network.fit(iter.next());
    assertEquals(7, network.getLayerWiseConfigurations().getIterationCount());

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ModelSerializer.writeModel(network, baos, true);
    byte[] asBytes = baos.toByteArray();

    ByteArrayInputStream bais = new ByteArrayInputStream(asBytes);
    MultiLayerNetwork net = ModelSerializer.restoreMultiLayerNetwork(bais, true);
    assertEquals(7, net.getLayerWiseConfigurations().getIterationCount());
}
 
Example #25
Source File: BNGradientCheckTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
    public void testGradient2dFixedGammaBeta() {
        DataNormalization scaler = new NormalizerMinMaxScaler();
        DataSetIterator iter = new IrisDataSetIterator(150, 150);
        scaler.fit(iter);
        iter.setPreProcessor(scaler);
        DataSet ds = iter.next();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        for (boolean useLogStd : new boolean[]{true, false}) {
            MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().updater(new NoOp())
                    .dataType(DataType.DOUBLE)
                    .seed(12345L)
                    .dist(new NormalDistribution(0, 1)).list()
                    .layer(0, new DenseLayer.Builder().nIn(4).nOut(3).activation(Activation.IDENTITY).build())
                    .layer(1, new BatchNormalization.Builder().useLogStd(useLogStd).lockGammaBeta(true).gamma(2.0).beta(0.5).nOut(3)
                            .build())
                    .layer(2, new ActivationLayer.Builder().activation(Activation.TANH).build())
                    .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                            .activation(Activation.SOFTMAX).nIn(3).nOut(3).build());

            MultiLayerNetwork mln = new MultiLayerNetwork(builder.build());
            mln.init();

//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());

            //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
            //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
            //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
            Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "1_log10stdev"));
            boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(mln).input(input)
                    .labels(labels).excludeParams(excludeParams));

            assertTrue(gradOK);
            TestUtils.testModelSerialization(mln);
        }
    }
 
Example #26
Source File: ComputationGraphTestRNN.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testTBPTTLongerThanTS() {
    int tbpttLength = 100;
    int timeSeriesLength = 20;
    int miniBatchSize = 7;
    int nIn = 5;
    int nOut = 4;

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).graphBuilder()
                    .addInputs("in")
                    .addLayer("0", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn).nOut(7)
                                    .activation(Activation.TANH)
                                    .dist(new NormalDistribution(0, 0.5)).build(), "in")
                    .addLayer("1", new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(7).nOut(8)
                                    .activation(Activation.TANH)
                                    .dist(new NormalDistribution(0,
                                                    0.5))
                                    .build(), "0")
                    .addLayer("out", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                    .nIn(8).nOut(nOut)
                                    .activation(Activation.SOFTMAX)
                                    .dist(new NormalDistribution(0, 0.5)).build(), "1")
                    .setOutputs("out").backpropType(BackpropType.TruncatedBPTT)
                    .tBPTTBackwardLength(tbpttLength).tBPTTForwardLength(tbpttLength).build();

    Nd4j.getRandom().setSeed(12345);
    ComputationGraph graph = new ComputationGraph(conf);
    graph.init();

    INDArray inputLong = Nd4j.rand(new int[] {miniBatchSize, nIn, timeSeriesLength});
    INDArray labelsLong = Nd4j.rand(new int[] {miniBatchSize, nOut, timeSeriesLength});

    INDArray initialParams = graph.params().dup();
    graph.fit(new INDArray[] {inputLong}, new INDArray[] {labelsLong});
    INDArray afterParams = graph.params();

    assertNotEquals(initialParams, afterParams);
}
 
Example #27
Source File: TestVAE.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testJsonYaml() {

    MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().seed(12345).list()
                    .layer(0, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.IDENTITY))
                                    .nIn(3).nOut(4).encoderLayerSizes(5).decoderLayerSizes(6).build())
                    .layer(1, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new GaussianReconstructionDistribution(Activation.TANH))
                                    .nIn(7).nOut(8).encoderLayerSizes(9).decoderLayerSizes(10).build())
                    .layer(2, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new BernoulliReconstructionDistribution()).nIn(11)
                                    .nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(3, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new ExponentialReconstructionDistribution(Activation.TANH))
                                    .nIn(11).nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(4, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .lossFunction(new ActivationTanH(), LossFunctions.LossFunction.MSE).nIn(11)
                                    .nOut(12).encoderLayerSizes(13).decoderLayerSizes(14).build())
                    .layer(5, new org.deeplearning4j.nn.conf.layers.variational.VariationalAutoencoder.Builder()
                                    .reconstructionDistribution(new CompositeReconstructionDistribution.Builder()
                                                    .addDistribution(5, new GaussianReconstructionDistribution())
                                                    .addDistribution(5,
                                                                    new GaussianReconstructionDistribution(Activation.TANH))
                                                    .addDistribution(5, new BernoulliReconstructionDistribution())
                                                    .build())
                                    .nIn(15).nOut(16).encoderLayerSizes(17).decoderLayerSizes(18).build())
                    .layer(1, new OutputLayer.Builder().lossFunction(LossFunctions.LossFunction.MSE).nIn(18)
                                    .nOut(19).activation(new ActivationTanH()).build())
                    .build();

    String asJson = config.toJson();
    String asYaml = config.toYaml();

    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(asJson);
    MultiLayerConfiguration fromYaml = MultiLayerConfiguration.fromYaml(asYaml);

    assertEquals(config, fromJson);
    assertEquals(config, fromYaml);
}
 
Example #28
Source File: EvalTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvaluativeListenerSimple(){
    //Sanity check: https://github.com/deeplearning4j/deeplearning4j/issues/5351

    // Network config
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()

            .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).seed(42)
            .updater(new Sgd(1e-6)).list()
            .layer(0, new DenseLayer.Builder().nIn(4).nOut(2).activation(Activation.TANH)
                    .weightInit(WeightInit.XAVIER).build())
            .layer(1, new org.deeplearning4j.nn.conf.layers.OutputLayer.Builder(
                    LossFunctions.LossFunction.MCXENT).nIn(2).nOut(3).weightInit(WeightInit.XAVIER)
                    .activation(Activation.SOFTMAX).build())
            .build();

    // Instantiate model
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    // Train-test split
    DataSetIterator iter = new IrisDataSetIterator(30, 150);
    DataSetIterator iterTest = new IrisDataSetIterator(30, 150);

    net.setListeners(new EvaluativeListener(iterTest, 3));

    for( int i=0; i<3; i++ ){
        net.fit(iter);
    }
}
 
Example #29
Source File: TestLrChanges.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testChangeLSGD() {
    //Simple test for no updater nets
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .activation(Activation.TANH)
            .seed(12345)
            .updater(new Sgd(0.1))
            .list()
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
            .layer(new OutputLayer.Builder().nIn(10).nOut(10).lossFunction(LossFunctions.LossFunction.MSE).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
    net.setLearningRate(1.0);
    net.setLearningRate(1, 0.5);
    assertEquals(1.0, net.getLearningRate(0), 0.0);
    assertEquals(0.5, net.getLearningRate(1), 0.0);


    ComputationGraph cg = net.toComputationGraph();
    cg.setLearningRate(2.0);
    cg.setLearningRate("1", 2.5);
    assertEquals(2.0, cg.getLearningRate("0"), 0.0);
    assertEquals(2.5, cg.getLearningRate("1"), 0.0);

}
 
Example #30
Source File: CNNGradientCheckTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
    public void testGradientCNNMLN() {
        if(this.format != CNN2DFormat.NCHW) //Only test NCHW due to flat input format...
            return;

        //Parameterized test, testing combinations of:
        // (a) activation function
        // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
        // (c) Loss function (with specified output activations)
        Activation[] activFns = {Activation.SIGMOID, Activation.TANH};
        boolean[] characteristic = {false, true}; //If true: run some backprop steps first

        LossFunctions.LossFunction[] lossFunctions =
                {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD, LossFunctions.LossFunction.MSE};
        Activation[] outputActivations = {Activation.SOFTMAX, Activation.TANH}; //i.e., lossFunctions[i] used with outputActivations[i] here

        DataSet ds = new IrisDataSetIterator(150, 150).next();
        ds.normalizeZeroMeanZeroUnitVariance();
        INDArray input = ds.getFeatures();
        INDArray labels = ds.getLabels();

        for (Activation afn : activFns) {
            for (boolean doLearningFirst : characteristic) {
                for (int i = 0; i < lossFunctions.length; i++) {
                    LossFunctions.LossFunction lf = lossFunctions[i];
                    Activation outputActivation = outputActivations[i];

                    MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder()
                            .dataType(DataType.DOUBLE)
                            .optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).updater(new NoOp())
                            .weightInit(WeightInit.XAVIER).seed(12345L).list()
                            .layer(0, new ConvolutionLayer.Builder(1, 1).nOut(6).activation(afn).build())
                            .layer(1, new OutputLayer.Builder(lf).activation(outputActivation).nOut(3).build())
                            .setInputType(InputType.convolutionalFlat(1, 4, 1));

                    MultiLayerConfiguration conf = builder.build();

                    MultiLayerNetwork mln = new MultiLayerNetwork(conf);
                    mln.init();
                    String name = new Object() {
                    }.getClass().getEnclosingMethod().getName();

                    if (doLearningFirst) {
                        //Run a number of iterations of learning
                        mln.setInput(ds.getFeatures());
                        mln.setLabels(ds.getLabels());
                        mln.computeGradientAndScore();
                        double scoreBefore = mln.score();
                        for (int j = 0; j < 10; j++)
                            mln.fit(ds);
                        mln.computeGradientAndScore();
                        double scoreAfter = mln.score();
                        //Can't test in 'characteristic mode of operation' if not learning
                        String msg = name + " - score did not (sufficiently) decrease during learning - activationFn="
                                + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                                + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                                + ", scoreAfter=" + scoreAfter + ")";
                        assertTrue(msg, scoreAfter < 0.9 * scoreBefore);
                    }

                    if (PRINT_RESULTS) {
                        System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf + ", outputActivation="
                                + outputActivation + ", doLearningFirst=" + doLearningFirst);
//                        for (int j = 0; j < mln.getnLayers(); j++)
//                            System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
                    }

                    boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(gradOK);
                    TestUtils.testModelSerialization(mln);
                }
            }
        }
    }