Java Code Examples for org.deeplearning4j.nn.conf.NeuralNetConfiguration

The following examples show how to use org.deeplearning4j.nn.conf.NeuralNetConfiguration. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: deeplearning4j   Source File: CustomPreprocessorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
    public void testCustomPreprocessor() {
        //Second: let's create a MultiLayerCofiguration with one, and check JSON and YAML config actually works...
        MultiLayerConfiguration conf =
                        new NeuralNetConfiguration.Builder().list()
                                        .layer(0, new DenseLayer.Builder().nIn(10).nOut(10).build())
                                        .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).nIn(10)
                                                .activation(Activation.SOFTMAX).nOut(10).build())
                                        .inputPreProcessor(0, new MyCustomPreprocessor())
                                        .build();

        String json = conf.toJson();
        String yaml = conf.toYaml();

//        System.out.println(json);

        MultiLayerConfiguration confFromJson = MultiLayerConfiguration.fromJson(json);
        assertEquals(conf, confFromJson);

        MultiLayerConfiguration confFromYaml = MultiLayerConfiguration.fromYaml(yaml);
        assertEquals(conf, confFromYaml);

        assertTrue(confFromJson.getInputPreProcess(0) instanceof MyCustomPreprocessor);

    }
 
Example 2
Source Project: deeplearning4j   Source File: WeightInitIdentityTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test identity mapping for 1d convolution
 */
@Test
public void testIdConv1D() {
    final INDArray input = Nd4j.randn(DataType.FLOAT, 1,5,7);
    final String inputName = "input";
    final String conv = "conv";
    final String output = "output";
    final ComputationGraph graph = new ComputationGraph(new NeuralNetConfiguration.Builder()
            .graphBuilder()
            .setInputTypes(InputType.inferInputType(input))
            .addInputs(inputName)
            .setOutputs(output)
            .layer(conv, new Convolution1DLayer.Builder(7)
                    .convolutionMode(ConvolutionMode.Same)
                    .nOut(input.size(1))
                    .weightInit(new WeightInitIdentity())
                    .activation(new ActivationIdentity())
                    .build(), inputName)
            .layer(output, new RnnLossLayer.Builder().activation(new ActivationIdentity()).build(), conv)
            .build());
    graph.init();

    assertEquals("Mapping was not identity!", input, graph.outputSingle(input).reshape(input.shape()));
}
 
Example 3
Source Project: deeplearning4j   Source File: FrozenLayerWithBackprop.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public org.deeplearning4j.nn.api.Layer instantiate(NeuralNetConfiguration conf,
                                                   Collection<TrainingListener> trainingListeners, int layerIndex, INDArray layerParamsView,
                                                   boolean initializeParams, DataType networkDataType) {

    //Need to be able to instantiate a layer, from a config - for JSON -> net type situations
    org.deeplearning4j.nn.api.Layer underlying = getUnderlying().instantiate(getInnerConf(conf), trainingListeners,
                    layerIndex, layerParamsView, initializeParams, networkDataType);

    NeuralNetConfiguration nncUnderlying = underlying.conf();

    if (nncUnderlying.variables() != null) {
        List<String> vars = nncUnderlying.variables(true);
        nncUnderlying.clearVariables();
        conf.clearVariables();
        for (String s : vars) {
            conf.variables(false).add(s);
            nncUnderlying.variables(false).add(s);
        }
    }

    return new org.deeplearning4j.nn.layers.FrozenLayerWithBackprop(underlying);
}
 
Example 4
Source Project: deeplearning4j   Source File: ConvolutionLayerSetupTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiChannel() throws Exception {
    INDArray in = Nd4j.rand(new int[] {10, 3, 28, 28});
    INDArray labels = Nd4j.rand(10, 2);
    DataSet next = new DataSet(in, labels);

    NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLFW();
    builder.setInputType(InputType.convolutional(28, 28, 3));
    MultiLayerConfiguration conf = builder.build();
    ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(2).getLayer();
    assertEquals(6, layer2.getNIn());

    MultiLayerNetwork network = new MultiLayerNetwork(conf);
    network.init();
    network.fit(next);
}
 
Example 5
Source Project: deeplearning4j   Source File: TestListeners.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListenersGraph() {
    TestListener.clearCounts();

    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().graphBuilder()
                    .addInputs("in").addLayer("0",
                                    new OutputLayer.Builder(LossFunctions.LossFunction.MSE).nIn(10).nOut(10)
                                                    .activation(Activation.TANH).build(),
                                    "in")
                    .setOutputs("0").build();

    ComputationGraph model = new ComputationGraph(conf);
    model.init();

    testListenersForModel(model, Collections.singletonList(new TestListener()));
}
 
Example 6
Source Project: deeplearning4j   Source File: ModelSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteMLNModel() throws Exception {
    int nIn = 5;
    int nOut = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01)
                    .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build())
                    .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder()
                                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    File tempFile = tempDir.newFile();

    ModelSerializer.writeModel(net, tempFile, true);

    MultiLayerNetwork network = ModelSerializer.restoreMultiLayerNetwork(tempFile);

    assertEquals(network.getLayerWiseConfigurations().toJson(), net.getLayerWiseConfigurations().toJson());
    assertEquals(net.params(), network.params());
    assertEquals(net.getUpdater().getStateViewArray(), network.getUpdater().getStateViewArray());
}
 
Example 7
Source Project: deeplearning4j   Source File: SameDiffCustomLayerTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInputValidationSameDiffVertex(){
    final ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder().graphBuilder()
            .addVertex("a", new ValidatingSameDiffVertex(), "input")
            .addLayer("output", new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build(), "a")
            .addInputs("input")
            .setInputTypes(InputType.feedForward(2))
            .setOutputs("output")
            .build();

    final ComputationGraph net = new ComputationGraph(config);
    net.init();

    final INDArray goodInput = Nd4j.rand(1, 2);
    final INDArray badInput = Nd4j.rand(2, 2);

    net.fit(new INDArray[]{goodInput}, new INDArray[]{goodInput});

    exceptionRule.expect(IllegalArgumentException.class);
    exceptionRule.expectMessage("Expected Message");
    net.fit(new INDArray[]{badInput}, new INDArray[]{badInput});
}
 
Example 8
Source Project: deeplearning4j   Source File: SameDiffCustomLayerTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInputValidationSameDiffLayer(){
    final MultiLayerConfiguration config = new NeuralNetConfiguration.Builder().list()
            .layer(new ValidatingSameDiffLayer())
            .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MSE).activation(Activation.SIGMOID).nOut(2).build())
            .setInputType(InputType.feedForward(2))
            .build();

    final MultiLayerNetwork net = new MultiLayerNetwork(config);
    net.init();

    final INDArray goodInput = Nd4j.rand(1, 2);
    final INDArray badInput = Nd4j.rand(2, 2);

    net.fit(goodInput, goodInput);

   exceptionRule.expect(IllegalArgumentException.class);
   exceptionRule.expectMessage("Expected Message");
   net.fit(badInput, badInput);
}
 
Example 9
Source Project: deeplearning4j   Source File: LayerConfigTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUpdaterAdamParamsLayerwiseOverride() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1.0, 0.5, 0.5, 1e-8))
            .list()
            .layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
                    .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).updater(new Adam(1.0, 0.6, 0.7, 1e-8)).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.6, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta1(), 0.0);
    assertEquals(0.5, ((Adam) ((BaseLayer) conf.getConf(0).getLayer()).getIUpdater()).getBeta2(), 0.0);
    assertEquals(0.7, ((Adam) ((BaseLayer) conf.getConf(1).getLayer()).getIUpdater()).getBeta2(), 0.0);
}
 
Example 10
Source Project: deeplearning4j   Source File: SameDiffVertex.java    License: Apache License 2.0 6 votes vote down vote up
public void applyGlobalConfig(NeuralNetConfiguration.Builder b) {
    if(regularization == null || regularization.isEmpty()){
        regularization = b.getRegularization();
    }
    if(regularizationBias == null || regularizationBias.isEmpty()){
        regularizationBias = b.getRegularizationBias();
    }
    if (updater == null) {
        updater = b.getIUpdater();
    }
    if (biasUpdater == null) {
        biasUpdater = b.getBiasUpdater();
    }
    if (gradientNormalization == null) {
        gradientNormalization = b.getGradientNormalization();
    }
    if (Double.isNaN(gradientNormalizationThreshold)) {
        gradientNormalizationThreshold = b.getGradientNormalizationThreshold();
    }

    applyGlobalConfigToLayer(b);
}
 
Example 11
Source Project: deeplearning4j   Source File: TestUpdaters.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSetGetUpdater2() {
    //Same as above test, except that we are doing setUpdater on a new network
    Nd4j.getRandom().setSeed(12345L);
    double lr = 0.03;
    int nIn = 4;
    int nOut = 8;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().updater(new Nesterovs(lr,0.6)).list()
                    .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(5)
                                    .updater(org.deeplearning4j.nn.conf.Updater.SGD).build())
                    .layer(1, new DenseLayer.Builder().nIn(5).nOut(6)
                                    .updater(new NoOp()).build())
                    .layer(2, new DenseLayer.Builder().nIn(6).nOut(7)
                                    .updater(org.deeplearning4j.nn.conf.Updater.ADAGRAD).build())
                    .layer(3, new OutputLayer.Builder().nIn(7).nOut(nOut).activation(Activation.SOFTMAX)
                                    .updater(org.deeplearning4j.nn.conf.Updater.NESTEROVS).build())
                    .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    Updater newUpdater = UpdaterCreator.getUpdater(net);
    net.setUpdater(newUpdater);
    assertTrue(newUpdater == net.getUpdater()); //Should be identical object
}
 
Example 12
Source Project: deeplearning4j   Source File: TestParallelEarlyStopping.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(1.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(10, 150);
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(10)) //Initial score is ~2.5
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer =
                    new EarlyStoppingParallelTrainer<>(esConf, net, irisIter, null, 2, 2, 1);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());

    assertTrue(result.getBestModelEpoch() <= 0);
    assertNotNull(result.getBestModel());
}
 
Example 13
Source Project: deeplearning4j   Source File: GravesLSTM.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Layer instantiate(NeuralNetConfiguration conf, Collection<TrainingListener> trainingListeners,
                         int layerIndex, INDArray layerParamsView, boolean initializeParams, DataType networkDataType) {
    LayerValidation.assertNInNOutSet("GravesLSTM", getLayerName(), layerIndex, getNIn(), getNOut());
    org.deeplearning4j.nn.layers.recurrent.GravesLSTM ret =
                    new org.deeplearning4j.nn.layers.recurrent.GravesLSTM(conf, networkDataType);
    ret.setListeners(trainingListeners);
    ret.setIndex(layerIndex);
    ret.setParamsViewArray(layerParamsView);
    Map<String, INDArray> paramTable = initializer().init(conf, layerParamsView, initializeParams);
    ret.setParamTable(paramTable);
    ret.setConf(conf);
    return ret;
}
 
Example 14
Source Project: deeplearning4j   Source File: CenterLossOutputLayerTest.java    License: Apache License 2.0 5 votes vote down vote up
public ComputationGraph getCNNMnistConfig() {

        int nChannels = 1; // Number of input channels
        int outputNum = 10; // The number of possible outcomes

        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345) // Training iterations as above
                        .l2(0.0005).weightInit(WeightInit.XAVIER)
                        .updater(new Nesterovs(0.01, 0.9))
                        .graphBuilder().addInputs("input")
                        .setInputTypes(InputType.convolutionalFlat(28, 28, 1))
                        .addLayer("0", new ConvolutionLayer.Builder(5, 5)
                                        //nIn and nOut specify channels. nIn here is the nChannels and nOut is the number of filters to be applied
                                        .nIn(nChannels).stride(1, 1).nOut(20).activation(Activation.IDENTITY).build(),
                                        "input")
                        .addLayer("1", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "0")
                        .addLayer("2", new ConvolutionLayer.Builder(5, 5)
                                        //Note that nIn need not be specified in later layers
                                        .stride(1, 1).nOut(50).activation(Activation.IDENTITY).build(), "1")
                        .addLayer("3", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX).kernelSize(2, 2)
                                        .stride(2, 2).build(), "2")
                        .addLayer("4", new DenseLayer.Builder().activation(Activation.RELU).nOut(500).build(), "3")
                        .addLayer("output",
                                        new org.deeplearning4j.nn.conf.layers.CenterLossOutputLayer.Builder(
                                                        LossFunction.MCXENT).nOut(outputNum)
                                                                        .activation(Activation.SOFTMAX).build(),
                                        "4")
                        .setOutputs("output").build();

        ComputationGraph graph = new ComputationGraph(conf);
        graph.init();

        return graph;
    }
 
Example 15
@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    FrozenLayerWithBackprop fl = (FrozenLayerWithBackprop) conf.getLayer();
    Layer innerLayer = fl.getUnderlying();
    ParamInitializer initializer = innerLayer.initializer();
    conf.setLayer(innerLayer);
    Map<String, INDArray> m = initializer.getGradientsFromFlattened(conf, gradientView);
    conf.setLayer(fl);
    return m;
}
 
Example 16
Source Project: deeplearning4j   Source File: MiscRegressionTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFrozenNewFormat(){
    MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder()
            .list()
            .layer(0, new FrozenLayer(new DenseLayer.Builder().nIn(10).nOut(10).build()))
            .build();

    String json = configuration.toJson();
    MultiLayerConfiguration fromJson = MultiLayerConfiguration.fromJson(json);
    assertEquals(configuration, fromJson);
}
 
Example 17
Source Project: deeplearning4j   Source File: LocallyConnected1D.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void applyGlobalConfigToLayer(NeuralNetConfiguration.Builder globalConfig) {
    if (activation == null) {
        activation = SameDiffLayerUtils.fromIActivation(globalConfig.getActivationFn());
    }
    if (cm == null) {
        cm = globalConfig.getConvolutionMode();
    }
}
 
Example 18
Source Project: jstarcraft-rns   Source File: DeepFMParameter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, INDArray> init(NeuralNetConfiguration configuration, INDArray view, boolean initialize) {
    Map<String, INDArray> parameters = Collections.synchronizedMap(new LinkedHashMap<String, INDArray>());
    FeedForwardLayer layerConfiguration = (FeedForwardLayer) configuration.getLayer();
    long numberOfOut = layerConfiguration.getNOut();
    long numberOfWeights = numberOfFeatures * numberOfOut;
    INDArray weight = view.get(new INDArrayIndex[] { NDArrayIndex.point(0), NDArrayIndex.interval(0, numberOfWeights) });
    INDArray bias = view.get(NDArrayIndex.point(0), NDArrayIndex.interval(numberOfWeights, numberOfWeights + numberOfOut));

    parameters.put(WEIGHT_KEY, this.createWeightMatrix(configuration, weight, initialize));
    parameters.put(BIAS_KEY, createBias(configuration, bias, initialize));
    configuration.addVariable(WEIGHT_KEY);
    configuration.addVariable(BIAS_KEY);
    return parameters;
}
 
Example 19
Source Project: deeplearning4j   Source File: OCNNParamInitializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, INDArray> init(NeuralNetConfiguration conf, INDArray paramsView, boolean initializeParams) {
    org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer ocnnOutputLayer = ( org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer) conf.getLayer();
    Map<String, INDArray> params = Collections.synchronizedMap(new LinkedHashMap<String, INDArray>());
    val nIn = ocnnOutputLayer.getNIn();
    int hiddenLayer = ocnnOutputLayer.getHiddenSize();
    Preconditions.checkState(hiddenLayer > 0, "OCNNOutputLayer hidden layer state: must be non-zero.");

    val firstLayerWeightLength =  hiddenLayer;
    val secondLayerLength = nIn * hiddenLayer;
    int rLength = 1;
    INDArray weightView = paramsView.get(point(0),interval(0, firstLayerWeightLength))
            .reshape(1,hiddenLayer);
    INDArray weightsTwoView = paramsView.get(point(0),
            NDArrayIndex.interval(firstLayerWeightLength,
                    firstLayerWeightLength + secondLayerLength))
            .reshape('f',nIn,hiddenLayer);
    INDArray rView = paramsView.get(point(0),point(paramsView.length() - rLength));


    INDArray paramViewPut = createWeightMatrix(conf, weightView, initializeParams);
    params.put(W_KEY, paramViewPut);
    conf.addVariable(W_KEY);
    INDArray paramIvewPutTwo = createWeightMatrix(conf,weightsTwoView,initializeParams);
    params.put(V_KEY,paramIvewPutTwo);
    conf.addVariable(V_KEY);
    INDArray rViewPut = createWeightMatrix(conf,rView,initializeParams);
    params.put(R_KEY,rViewPut);
    conf.addVariable(R_KEY);

    return params;
}
 
Example 20
Source Project: jstarcraft-rns   Source File: DeepFMInputConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Layer instantiate(NeuralNetConfiguration configuration, Collection<TrainingListener> monitors, int layerIndex, INDArray parameters, boolean initialize) {
    DeepFMInputLayer layer = new DeepFMInputLayer(configuration, dimensionSizes);
    layer.setListeners(monitors);
    layer.setIndex(layerIndex);
    layer.setParamsViewArray(parameters);
    Map<String, INDArray> table = initializer().init(configuration, parameters, initialize);
    layer.setParamTable(table);
    layer.setConf(configuration);
    return layer;
}
 
Example 21
Source Project: deeplearning4j   Source File: TestEarlyStopping.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadTuning() {
    //Test poor tuning (high LR): should terminate on MaxScoreIterationTerminationCondition

    Nd4j.getRandom().setSeed(12345);
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd(5.0)) //Intentionally huge LR
                    .weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(1));

    DataSetIterator irisIter = new IrisDataSetIterator(150, 150);
    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5000))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(1, TimeUnit.MINUTES),
                                                    new MaxScoreIterationTerminationCondition(10)) //Initial score is ~2.5
                                    .scoreCalculator(new DataSetLossCalculator(irisIter, true)).modelSaver(saver)
                                    .build();

    IEarlyStoppingTrainer trainer = new EarlyStoppingTrainer(esConf, net, irisIter);
    EarlyStoppingResult result = trainer.fit();

    assertTrue(result.getTotalEpochs() < 5);
    assertEquals(EarlyStoppingResult.TerminationReason.IterationTerminationCondition,
                    result.getTerminationReason());
    String expDetails = new MaxScoreIterationTerminationCondition(10).toString();
    assertEquals(expDetails, result.getTerminationDetails());

    assertEquals(0, result.getBestModelEpoch());
    assertNotNull(result.getBestModel());
}
 
Example 22
Source Project: deeplearning4j   Source File: TestFailureListener.java    License: Apache License 2.0 5 votes vote down vote up
@Ignore
@Test
public void testFailureRandom_AND() throws Exception {

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .updater(new Adam(1e-4))
            .list()
            .layer(0, new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX).lossFunction(LossFunctions.LossFunction.MCXENT).build())
            .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    String hostname = InetAddress.getLocalHost().getHostName();
    assertNotNull(hostname);
    assertFalse(hostname.isEmpty());

    net.setListeners(new FailureTestingListener(
            FailureTestingListener.FailureMode.ILLEGAL_STATE,
            new FailureTestingListener.And(
                    new FailureTestingListener.HostNameTrigger(hostname),
                    new FailureTestingListener.RandomProb(FailureTestingListener.CallType.ANY, 0.05))
    ));

    DataSetIterator iter = new IrisDataSetIterator(5,150);

    net.fit(iter);
}
 
Example 23
Source Project: deeplearning4j   Source File: LayerConfigValidationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWeightInitDistNotSet() {
    // Warning thrown only since global dist can be set with a different weight init locally
    MultiLayerConfiguration conf =
            new NeuralNetConfiguration.Builder().updater(new Sgd(0.3)).dist(new GaussianDistribution(1e-3, 2))
                    .list().layer(0, new DenseLayer.Builder().nIn(2).nOut(2).build())
                    .layer(1, new DenseLayer.Builder().nIn(2).nOut(2).build()).build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();
}
 
Example 24
Source Project: deeplearning4j   Source File: BatchNormalizationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void checkSerialization() throws Exception {
    //Serialize the batch norm network (after training), and make sure we get same activations out as before
    // i.e., make sure state is properly stored

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .seed(12345)
            .list()
            .layer(0, new ConvolutionLayer.Builder().nIn(1).nOut(6).weightInit(WeightInit.XAVIER)
                    .activation(Activation.IDENTITY).build())
            .layer(1, new BatchNormalization.Builder().build())
            .layer(2, new ActivationLayer.Builder().activation(Activation.LEAKYRELU).build())
            .layer(3, new DenseLayer.Builder().nOut(10).activation(Activation.LEAKYRELU).build())
            .layer(4, new BatchNormalization.Builder().build())
            .layer(5, new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).nOut(10).build())
            .setInputType(InputType.convolutionalFlat(28, 28, 1)).build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    DataSetIterator iter = new MnistDataSetIterator(16, true, 12345);
    for (int i = 0; i < 20; i++) {
        net.fit(iter.next());
    }

    INDArray in = iter.next().getFeatures();

    INDArray out = net.output(in, false);
    INDArray out2 = net.output(in, false);

    assertEquals(out, out2);

    MultiLayerNetwork net2 = TestUtils.testModelSerialization(net);

    INDArray outDeser = net2.output(in, false);

    assertEquals(out, outDeser);
}
 
Example 25
private static Model buildModel() throws Exception {

    final int numInputs = 3;
    final int numOutputs = 2;

    final MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
        .list(
            new OutputLayer.Builder()
                           .nIn(numInputs)
                           .nOut(numOutputs)
                           .activation(Activation.IDENTITY)
                           .lossFunction(LossFunctions.LossFunction.MSE)
                           .build()
            )
        .build();

    final MultiLayerNetwork model = new MultiLayerNetwork(conf);
    model.init();

    final float[] floats = new float[]{ +1, +1, +1, -1, -1, -1, 0, 0 };
    // positive weight for first output, negative weight for second output, no biases
    assertEquals((numInputs+1)*numOutputs, floats.length);

    final INDArray params = Nd4j.create(floats);
    model.setParams(params);

    return model;
  }
 
Example 26
Source Project: deeplearning4j   Source File: ModelSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidLoading2() throws Exception {
    int nIn = 5;
    int nOut = 6;

    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).l1(0.01)
            .l2(0.01).updater(new Sgd(0.1)).activation(Activation.TANH).weightInit(WeightInit.XAVIER).list()
            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(20).build())
            .layer(1, new DenseLayer.Builder().nIn(20).nOut(30).build()).layer(2, new OutputLayer.Builder()
                    .lossFunction(LossFunctions.LossFunction.MSE).nIn(30).nOut(nOut).build())
            .build();

    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.init();

    File tempFile = tempDir.newFile("testInvalidLoading2.bin");

    ModelSerializer.writeModel(net, tempFile, true);

    try {
        ModelSerializer.restoreComputationGraph(tempFile);
        fail();
    } catch (Exception e){
        String msg = e.getMessage();
        assertTrue(msg, msg.contains("JSON") && msg.contains("restoreMultiLayerNetwork"));
    }
}
 
Example 27
@Test
@Ignore    //Ignored 2019/04/09 - low priority: https://github.com/deeplearning4j/deeplearning4j/issues/6656
public void testVaePretrainSimpleCG() {
    //Simple sanity check on pretraining
    int nIn = 8;

    Nd4j.getRandom().setSeed(12345);
    ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).updater(new RmsProp())
                    .weightInit(WeightInit.XAVIER).graphBuilder().addInputs("in")
                    .addLayer("0", new VariationalAutoencoder.Builder().nIn(8).nOut(10).encoderLayerSizes(12)
                                    .decoderLayerSizes(13).reconstructionDistribution(
                                                    new GaussianReconstructionDistribution(Activation.IDENTITY))
                                    .build(), "in")
                    .setOutputs("0").build();

    //Do training on Spark with one executor, for 3 separate minibatches
    int rddDataSetNumExamples = 10;
    int totalAveragings = 5;
    int averagingFrequency = 3;
    ParameterAveragingTrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(rddDataSetNumExamples)
                    .averagingFrequency(averagingFrequency).batchSizePerWorker(rddDataSetNumExamples)
                    .saveUpdater(true).workerPrefetchNumBatches(0).build();
    Nd4j.getRandom().setSeed(12345);
    SparkComputationGraph sparkNet = new SparkComputationGraph(sc, conf.clone(), tm);

    List<DataSet> trainData = new ArrayList<>();
    int nDataSets = numExecutors() * totalAveragings * averagingFrequency;
    for (int i = 0; i < nDataSets; i++) {
        trainData.add(new DataSet(Nd4j.rand(rddDataSetNumExamples, nIn), null));
    }

    JavaRDD<DataSet> data = sc.parallelize(trainData);

    sparkNet.fit(data);
}
 
Example 28
Source Project: deeplearning4j   Source File: GradientCheckTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
    public void testEmbeddingLayerPreluSimple() {
        Random r = new Random(12345);
        int nExamples = 5;
        INDArray input = Nd4j.zeros(nExamples, 1);
        INDArray labels = Nd4j.zeros(nExamples, 3);
        for (int i = 0; i < nExamples; i++) {
            input.putScalar(i, r.nextInt(4));
            labels.putScalar(new int[] {i, r.nextInt(3)}, 1.0);
        }

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().l2(0.2).l1(0.1)
                .dataType(DataType.DOUBLE)
                .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).seed(12345L)
                .list().layer(new EmbeddingLayer.Builder().nIn(4).nOut(3).weightInit(WeightInit.XAVIER)
                                .updater(new NoOp()).build())
                .layer(new PReLULayer.Builder().inputShape(3).sharedAxes(1).updater(new NoOp()).build())
                .layer(new OutputLayer.Builder(LossFunction.MCXENT).nIn(3).nOut(3)
                        .weightInit(WeightInit.XAVIER).dist(new NormalDistribution(0, 1))
                        .updater(new NoOp()).activation(Activation.SOFTMAX).build())
                .build();

        MultiLayerNetwork mln = new MultiLayerNetwork(conf);
        mln.init();

        if (PRINT_RESULTS) {
            System.out.println("testEmbeddingLayerSimple");
//            for (int j = 0; j < mln.getnLayers(); j++)
//                System.out.println("Layer " + j + " # params: " + mln.getLayer(j).numParams());
        }

        boolean gradOK = GradientCheckUtil.checkGradients(mln, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

        String msg = "testEmbeddingLayerSimple";
        assertTrue(msg, gradOK);
    }
 
Example 29
Source Project: deeplearning4j   Source File: TestEarlyStoppingSpark.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testListeners() {
    MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
                    .updater(new Sgd()).weightInit(WeightInit.XAVIER).list()
                    .layer(0, new OutputLayer.Builder().nIn(4).nOut(3)
                                    .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                    .build();
    MultiLayerNetwork net = new MultiLayerNetwork(conf);
    net.setListeners(new ScoreIterationListener(5));


    JavaRDD<DataSet> irisData = getIris();

    EarlyStoppingModelSaver<MultiLayerNetwork> saver = new InMemoryModelSaver<>();
    EarlyStoppingConfiguration<MultiLayerNetwork> esConf =
                    new EarlyStoppingConfiguration.Builder<MultiLayerNetwork>()
                                    .epochTerminationConditions(new MaxEpochsTerminationCondition(5))
                                    .iterationTerminationConditions(
                                                    new MaxTimeIterationTerminationCondition(2, TimeUnit.MINUTES))
                                    .scoreCalculator(new SparkDataSetLossCalculator(irisData, true, sc.sc()))
                                    .modelSaver(saver).build();

    LoggingEarlyStoppingListener listener = new LoggingEarlyStoppingListener();

    IEarlyStoppingTrainer<MultiLayerNetwork> trainer = new SparkEarlyStoppingTrainer(
                    getContext().sc(), new ParameterAveragingTrainingMaster(true,
                                    Runtime.getRuntime().availableProcessors(), 1, 10, 1, 0),
                    esConf, net, irisData);
    trainer.setListener(listener);

    trainer.fit();

    assertEquals(1, listener.onStartCallCount);
    assertEquals(5, listener.onEpochCallCount);
    assertEquals(1, listener.onCompletionCallCount);
}
 
Example 30
Source Project: deeplearning4j   Source File: LSTMParamInitializer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, INDArray> getGradientsFromFlattened(NeuralNetConfiguration conf, INDArray gradientView) {
    org.deeplearning4j.nn.conf.layers.LSTM layerConf = (org.deeplearning4j.nn.conf.layers.LSTM) conf.getLayer();

    val nL = layerConf.getNOut(); //i.e., n neurons in this layer
    val nLast = layerConf.getNIn(); //i.e., n neurons in previous layer

    val length = numParams(conf);
    if (gradientView.length() != length)
        throw new IllegalStateException(
                        "Expected gradient view of length " + length + ", got length " + gradientView.length());

    val nParamsIn = nLast * (4 * nL);
    val nParamsRecurrent = nL * (4 * nL);
    val nBias = 4 * nL;
    INDArray inputWeightGradView = gradientView.get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(0, nParamsIn))
                    .reshape('f', nLast, 4 * nL);
    INDArray recurrentWeightGradView = gradientView
                    .get(NDArrayIndex.interval(0,0,true), NDArrayIndex.interval(nParamsIn, nParamsIn + nParamsRecurrent))
                    .reshape('f', nL, 4 * nL);
    INDArray biasGradView = gradientView.get(NDArrayIndex.interval(0,0,true),
                    NDArrayIndex.interval(nParamsIn + nParamsRecurrent, nParamsIn + nParamsRecurrent + nBias)); //already a row vector

    Map<String, INDArray> out = new LinkedHashMap<>();
    out.put(INPUT_WEIGHT_KEY, inputWeightGradView);
    out.put(RECURRENT_WEIGHT_KEY, recurrentWeightGradView);
    out.put(BIAS_KEY, biasGradView);

    return out;
}