Java Code Examples for org.nd4j.common.primitives.Pair#getFirst()

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);
    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{w.size(0), delta.size(0)}, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();

    //Normally we would clear weightNoiseParams here - but we want to reuse them for forward + backward + score
    // So this is instead done in MultiLayerNetwork/CompGraph backprop methods

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

@Override
public void call(Pair<List<VocabWord>, AtomicLong> pair) throws Exception {
    double numWordsSoFar = wordCount.getValue().doubleValue();

    List<VocabWord> sentence = pair.getFirst();
    double alpha2 = Math.max(minAlpha, alpha * (1 - (1.0 * numWordsSoFar / (double) totalWords)));
    int totalNewWords = 0;
    trainSentence(sentence, alpha2);
    totalNewWords += sentence.size();



    double newWords = totalNewWords + numWordsSoFar;
    double diff = Math.abs(newWords - lastChecked);
    if (diff >= 10000) {
        lastChecked = (int) newWords;
        log.info("Words so far " + newWords + " out of " + totalWords);
    }

    pair.getSecond().getAndAdd((long) totalNewWords);
}
 

@Test
public void testInverseComparison() {

    List<Pair<INDArray, String>> list = NDArrayCreationUtil.getAllTestMatricesWithShape(10, 10, 12345, DataType.DOUBLE);

    for (Pair<INDArray, String> p : list) {
        INDArray orig = p.getFirst();
        orig.assign(Nd4j.rand(orig.shape()));
        INDArray inverse = InvertMatrix.invert(orig, false);
        RealMatrix rm = CheckUtil.convertToApacheMatrix(orig);
        RealMatrix rmInverse = new LUDecomposition(rm).getSolver().getInverse();

        INDArray expected = CheckUtil.convertFromApacheMatrix(rmInverse, orig.dataType());
        assertTrue(p.getSecond(), CheckUtil.checkEntries(expected, inverse, 1e-3, 1e-4));
    }
}
 

/**
 * For use during inference. Will convert a given pair of a list of sentences to features and feature masks as appropriate.
 *
 * @param listOnlySentencePairs
 * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array
 */
public Pair<INDArray[], INDArray[]> featurizeSentencePairs(List<Pair<String, String>> listOnlySentencePairs) {
    Preconditions.checkState(sentencePairProvider != null, "The featurizeSentencePairs method is meant for inference with sentence pairs. Use only when the sentence pair provider is set (i.e not null).");

    List<Triple<String, String, String>> sentencePairsWithNullLabel = addDummyLabelForPairs(listOnlySentencePairs);
    SentencePairListProcessed sentencePairListProcessed = tokenizePairsMiniBatch(sentencePairsWithNullLabel);
    List<Pair<List<String>, String>> tokensAndLabelList = sentencePairListProcessed.getTokensAndLabelList();
    int outLength = sentencePairListProcessed.getMaxL();
    long[] segIdOnesFrom = sentencePairListProcessed.getSegIdOnesFrom();
    if (preProcessor != null) {
        Pair<INDArray[], INDArray[]> featuresAndMaskArraysPair = convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
        MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featuresAndMaskArraysPair.getFirst(), null, featuresAndMaskArraysPair.getSecond(), null);
        preProcessor.preProcess(dummyMDS);
        return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays());
    }
    return convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom);
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1)
    INDArray delta = pair.getSecond();

    // centers
    INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
    INDArray l = labels.castTo(centers.dataType());     //Ensure correct dtype (same as params); no-op if already correct dtype
    INDArray centersForExamples = l.mmul(centers);
    INDArray dLcdai = input.sub(centersForExamples);

    INDArray w = getParamWithNoise(CenterLossParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, w.dataType(), new long[]{w.size(0), delta.size(0)}, 'f');
    epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose();
    double lambda = layerConf().getLambda();
    epsilonNext.addi(dLcdai.muli(lambda)); // add center loss here

    weightNoiseParams.clear();

    return new Pair<>(pair.getFirst(), epsilonNext);
}
 

@Test
public void testExpandDims(){
    final List<Pair<INDArray, String>> testMatricesC = NDArrayCreationUtil.getAllTestMatricesWithShape('c', 3, 5, 0xDEAD, DataType.DOUBLE);
    final List<Pair<INDArray, String>> testMatricesF = NDArrayCreationUtil.getAllTestMatricesWithShape('f', 7, 11, 0xBEEF, DataType.DOUBLE);

    final ArrayList<Pair<INDArray, String>> testMatrices = new ArrayList<>(testMatricesC);
    testMatrices.addAll(testMatricesF);

    for (Pair<INDArray, String> testMatrixPair : testMatrices) {
        final String recreation = testMatrixPair.getSecond();
        final INDArray testMatrix = testMatrixPair.getFirst();
        final char ordering = testMatrix.ordering();
        val shape = testMatrix.shape();
        final int rank = testMatrix.rank();
        for (int i = -rank; i <= rank; i++) {
            final INDArray expanded = Nd4j.expandDims(testMatrix, i);

            final String message = "Expanding in Dimension " + i + "; Shape before expanding: " + Arrays.toString(shape) + " "+ordering+" Order; Shape after expanding: " + Arrays.toString(expanded.shape()) +  " "+expanded.ordering()+"; Input Created via: " + recreation;

            val tmR = testMatrix.ravel();
            val expR = expanded.ravel();
            assertEquals(message, 1, expanded.shape()[i < 0 ? i + rank : i]);
            assertEquals(message, tmR, expR);
            assertEquals(message, ordering,  expanded.ordering());

            testMatrix.assign(Nd4j.rand(DataType.DOUBLE, shape));
            assertEquals(message, testMatrix.ravel(), expanded.ravel());
        }
    }
}
 

/**
 * Constructor from parsed Keras layer configuration dictionary.
 *
 * @param layerConfig           dictionary containing Keras layer configuration
 * @param enforceTrainingConfig whether to enforce training-related configuration options
 * @param previousLayers dictionary containing the previous layers in the topology
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public KerasSimpleRnn(Map<String, Object> layerConfig, boolean enforceTrainingConfig,
                      Map<String, ? extends KerasLayer> previousLayers)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    super(layerConfig, enforceTrainingConfig);

    IWeightInit init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    IWeightInit recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    this.returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES());

    KerasRnnUtils.getRecurrentDropout(conf, layerConfig);
    this.unroll = KerasRnnUtils.getUnrollRecurrentLayer(conf, layerConfig);

    Pair<Boolean, Double> maskingConfig = KerasLayerUtils.getMaskingConfiguration(inboundLayerNames, previousLayers);

    LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint recurrentConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_RECURRENT_CONSTRAINT(), conf, kerasMajorVersion);

    SimpleRnn.Builder builder = new SimpleRnn.Builder()
            .name(this.layerName)
            .nOut(getNOutFromConfig(layerConfig, conf))
            .dropOut(this.dropout)
            .activation(getIActivationFromConfig(layerConfig, conf))
            .weightInit(init)
            .weightInitRecurrent(recurrentInit)
            .biasInit(0.0)
            .l1(this.weightL1Regularization)
            .l2(this.weightL2Regularization).dataFormat(RNNFormat.NWC);
    Integer nIn = KerasLayerUtils.getNInFromInputDim(layerConfig, conf);
    if(nIn != null)
        builder.setNIn(nIn);
    if (biasConstraint != null)
        builder.constrainBias(biasConstraint);
    if (weightConstraint != null)
        builder.constrainInputWeights(weightConstraint);
    if (recurrentConstraint != null)
        builder.constrainRecurrent(recurrentConstraint);

    this.layer = builder.build();
    if (!returnSequences) {
        this.layer = new LastTimeStep(this.layer);
    }
    if (maskingConfig.getFirst()) {
        this.layer = new MaskZeroLayer(this.layer, maskingConfig.getSecond());
    }
}
 

@Override
public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) {
    double score;
    long numParams;
    int numLayers;
    String modelConfigJson;
    int totalNumUpdates;
    if(candidate instanceof MultiLayerNetwork){
        MultiLayerNetwork m = (MultiLayerNetwork)candidate;
        score = m.score();
        numParams = m.numParams();
        numLayers = m.getnLayers();
        modelConfigJson = m.getLayerWiseConfigurations().toJson();
        totalNumUpdates = m.getLayerWiseConfigurations().getIterationCount();
    } else if(candidate instanceof ComputationGraph) {
        ComputationGraph cg = (ComputationGraph)candidate;
        score = cg.score();
        numParams = cg.numParams();
        numLayers = cg.getNumLayers();
        modelConfigJson = cg.getConfiguration().toJson();
        totalNumUpdates = cg.getConfiguration().getIterationCount();
    } else {
        score = 0;
        numParams = 0;
        numLayers = 0;
        totalNumUpdates = 0;
        modelConfigJson = "";
    }

    int idx = candidateInfo.getIndex();

    Pair<IntArrayList, FloatArrayList> pair = candidateScoreVsIter.computeIfAbsent(idx, k -> new Pair<>(new IntArrayList(), new FloatArrayList()));

    IntArrayList iter = pair.getFirst();
    FloatArrayList scores = pair.getSecond();

    //Do we need subsampling to avoid having too many data points?
    int subsamplingFreq = candidateScoreVsIterSubsampleFreq.computeIfAbsent(idx, k -> 1);
    if(iteration / subsamplingFreq > MAX_SCORE_VS_ITER_PTS){
        //Double subsampling frequency and re-parse data
        subsamplingFreq *= 2;
        candidateScoreVsIterSubsampleFreq.put(idx, subsamplingFreq);

        IntArrayList newIter = new IntArrayList();
        FloatArrayList newScores = new FloatArrayList();
        for( int i=0; i<iter.size(); i++ ){
            int it = iter.get(i);
            if(it % subsamplingFreq == 0){
                newIter.add(it);
                newScores.add(scores.get(i));
            }
        }

        iter = newIter;
        scores = newScores;
        candidateScoreVsIter.put(idx, new Pair<>(iter, scores));
    }

    if(iteration % subsamplingFreq == 0) {
        iter.add(iteration);
        scores.add((float) score);
    }


    int[] iters = iter.toIntArray();
    float[] fScores = new float[iters.length];
    for( int i=0; i<iters.length; i++ ){
        fScores[i] = scores.get(i);
    }

    ModelInfoPersistable p = new ModelInfoPersistable.Builder()
            .timestamp(candidateInfo.getCreatedTime())
            .sessionId(sessionId)
            .workerId(String.valueOf(candidateInfo.getIndex()))
            .modelIdx(candidateInfo.getIndex())
            .score(candidateInfo.getScore())
            .status(candidateInfo.getCandidateStatus())
            .scoreVsIter(iters, fScores)
            .lastUpdateTime(System.currentTimeMillis())
            .numParameters(numParams)
            .numLayers(numLayers)
            .totalNumUpdates(totalNumUpdates)
            .paramSpaceValues(candidateInfo.getFlatParams())
            .modelConfigJson(modelConfigJson)
            .exceptionStackTrace(candidateInfo.getExceptionStackTrace())
            .build();


    lastModelInfoPersistable.put(candidateInfo.getIndex(), p);
    statsStorage.putUpdate(p);
}
 

private static void testGravesBackwardBasicHelper(int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize,
                int timeSeriesLength) {

    INDArray inputData = Nd4j.ones(miniBatchSize, nIn, timeSeriesLength);

    NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder()
                    .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn)
                                    .nOut(lstmNHiddenUnits)
                                    .dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build())
                    .build();

    val numParams = conf.getLayer().initializer().numParams(conf);
    INDArray params = Nd4j.create(1, numParams);
    GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType());
    lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf)));
    //Set input, do a forward pass:
    lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces());
    assertNotNull(lstm.input());

    INDArray epsilon = Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength);

    Pair<Gradient, INDArray> out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces());
    Gradient outGradient = out.getFirst();
    INDArray nextEpsilon = out.getSecond();

    INDArray biasGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.BIAS_KEY);
    INDArray inWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY);
    INDArray recurrentWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY);
    assertNotNull(biasGradient);
    assertNotNull(inWeightGradient);
    assertNotNull(recurrentWeightGradient);

    assertArrayEquals(biasGradient.shape(), new long[] {1, 4 * lstmNHiddenUnits});
    assertArrayEquals(inWeightGradient.shape(), new long[] {nIn, 4 * lstmNHiddenUnits});
    assertArrayEquals(recurrentWeightGradient.shape(), new long[] {lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3});

    assertNotNull(nextEpsilon);
    assertArrayEquals(nextEpsilon.shape(), new long[] {miniBatchSize, nIn, timeSeriesLength});

    //Check update:
    for (String s : outGradient.gradientForVariable().keySet()) {
        lstm.update(outGradient.getGradientFor(s), s);
    }
}
 

public static void checkIntermediate(Map<String, INDArray> inputs, String modelName, String baseDir, String modelFileName,
                                         ExecuteWith execType, BiFunction<File,String,SameDiff> loader,
                                         Double maxRelErrorOverride, Double minAbsErrorOverride, File localTestDir, boolean printArraysDebugging) throws IOException {
        Preconditions.checkArgument((maxRelErrorOverride == null) == (minAbsErrorOverride == null), "Both maxRelErrorOverride and minAbsErrorOverride" +
                " must be null or both must be provided");
        Nd4j.EPS_THRESHOLD = 1e-3;
        OpExecOrderListener listener = new OpExecOrderListener();       //Used to collect exec order
        Pair<SameDiff, Map<String,INDArray>> p = getGraphAfterExec(baseDir, modelFileName, modelName, inputs, execType, loader, Collections.singletonList(listener), null, printArraysDebugging);
        SameDiff graph = p.getFirst();
        Map<String,INDArray> sdPredictions = p.getSecond();

        //Collect coverage info about ops
        OpValidation.collectTensorflowImportCoverage(graph);

        if (!execType.equals(ExecuteWith.JUST_PRINT)) {
            int count = 0;
            //Evaluate the nodes in their execution order - this is useful for debugging (as we want the *first* failure
            // to be detected before later failures)
            List<String> varNames = new ArrayList<>();
            Map<String,SameDiffOp> fns = graph.getOps();
            List<String> execOrder = listener.getOpNamesList();
            for(String opName : execOrder){
                String[] outputs = graph.getOutputsForOp(fns.get(opName).getOp());
                Collections.addAll(varNames, outputs);
            }

            for (String varName : varNames) {
                if (!inputs.containsKey(varName)) { //avoiding placeholders
                    INDArray tfValue = intermediateVars(modelName, baseDir, varName, localTestDir);
                    if (tfValue == null) {
                        continue;
                    }
                    log.info("Starting check: variable {}", varName);
                    if (skipNode(modelName, varName)) {
                        log.info("\n\tFORCING no check on " + varName);
                    } else {
                        assertArrayEquals("Shape not equal on node " + varName, tfValue.shape(), graph.getVariable(varName).getShape());
                        INDArray sdVal = sdPredictions.get(varName);
                        if(maxRelErrorOverride != null){
                            INDArray diff = Transforms.abs(tfValue.sub(sdVal), false);
                            INDArray absErrorMask = diff.gte(minAbsErrorOverride);   //value 1 if x[i] > minAbsError; value 0 otherwise. Used to get rid of 1e-30 vs. 1e-29 type failures
                            INDArray sumAbs = Transforms.abs(tfValue, true).addi(Transforms.abs(sdVal, true));
                            BooleanIndexing.replaceWhere(sumAbs, 1.0, Conditions.equals(0.0));  //Can only get 0.0 if both are zeros - need to avoid 0/0=NaN
                            INDArray relError = diff.divi(sumAbs);
                            relError.muli(absErrorMask);

                            int countExceeds = Nd4j.getExecutioner().exec(new MatchCondition(relError, Conditions.greaterThan(maxRelErrorOverride))).getInt(0);

                            double maxRE = -1;
                            //Mainly used for analysis in debugger:
                            DifferentialFunction op = null;
                            String[] opInputs = null;
                            if(countExceeds > 0){
                                maxRE = relError.maxNumber().doubleValue();
                                //Find the op that this variable is produced by
                                op = graph.getVariableOutputOp(varName);
                                opInputs = graph.getInputsForOp(op);
                            }


                            assertEquals( varName + ": " + countExceeds + " values exceed maxRelError=" + maxRelErrorOverride
                                    + " with minAbsError=" + minAbsErrorOverride + "; largest observed relError=" + maxRE, 0, countExceeds);
                        } else {
//                            assertEquals("Value not equal on node " + varName, tfValue, sdVal);
                            if(tfValue.equals(sdVal)){
                                System.out.println("Pass: " + varName);
                            } else {
                                System.out.println("FAIL: " + varName);
                                System.out.println("TF:\n" + tfValue);
                                System.out.println("SD:\n" + sdVal);
                            }

                        }
                        log.info("Values and shapes equal for {}", varName);
                        count++;
                    }

                }
            }

            assertTrue("No intermediate variables were checked", count > 0);
        }

        Nd4j.EPS_THRESHOLD = 1e-5;
    }
 

@Test
public void testBackwardIrisBasic() {
    ComputationGraphConfiguration configuration = getIrisGraphConfiguration();
    ComputationGraph graph = new ComputationGraph(configuration);
    graph.init();

    MultiLayerConfiguration mlc = getIrisMLNConfiguration();
    MultiLayerNetwork net = new MultiLayerNetwork(mlc);
    net.init();

    DataSetIterator iris = new IrisDataSetIterator(150, 150);
    DataSet ds = iris.next();

    //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs
    Nd4j.getRandom().setSeed(12345);
    int nParams = (4 * 5 + 5) + (5 * 3 + 3);
    INDArray params = Nd4j.rand(1, nParams);
    graph.setParams(params.dup());
    net.setParams(params.dup());

    INDArray input = ds.getFeatures();
    INDArray labels = ds.getLabels();
    graph.setInput(0, input.dup());
    graph.setLabel(0, labels.dup());

    net.setInput(input.dup());
    net.setLabels(labels.dup());

    //Compute gradients
    net.computeGradientAndScore();
    Pair<Gradient, Double> netGradScore = net.gradientAndScore();

    graph.computeGradientAndScore();
    Pair<Gradient, Double> graphGradScore = graph.gradientAndScore();

    assertEquals(netGradScore.getSecond(), graphGradScore.getSecond(), 1e-3);

    //Compare gradients
    Gradient netGrad = netGradScore.getFirst();
    Gradient graphGrad = graphGradScore.getFirst();

    assertNotNull(graphGrad);
    assertEquals(netGrad.gradientForVariable().size(), graphGrad.gradientForVariable().size());

    assertEquals(netGrad.getGradientFor("0_W"), graphGrad.getGradientFor("firstLayer_W"));
    assertEquals(netGrad.getGradientFor("0_b"), graphGrad.getGradientFor("firstLayer_b"));
    assertEquals(netGrad.getGradientFor("1_W"), graphGrad.getGradientFor("outputLayer_W"));
    assertEquals(netGrad.getGradientFor("1_b"), graphGrad.getGradientFor("outputLayer_b"));
}
 

@Test//(timeout = 25000L)
public void testOutputOnly() throws Exception {
    if(TFGraphTestZooModels.isPPC()){
        /*
        Ugly hack to temporarily disable tests on PPC only on CI
        Issue logged here: https://github.com/deeplearning4j/deeplearning4j/issues/7657
        These will be re-enabled for PPC once fixed - in the mean time, remaining tests will be used to detect and prevent regressions
         */

        log.warn("TEMPORARILY SKIPPING TEST ON PPC ARCHITECTURE DUE TO KNOWN JVM CRASH ISSUES - SEE https://github.com/deeplearning4j/deeplearning4j/issues/7657");
        OpValidationSuite.ignoreFailing();
    }


    Nd4j.create(1);

    for(String s : IGNORE_REGEXES){
        if(modelName.matches(s)){
            log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s);
            OpValidationSuite.ignoreFailing();
        }
    }
    Pair<Double,Double> precisionOverride = TFGraphTestAllHelper.testPrecisionOverride(modelName);
    Double maxRE = (precisionOverride == null ? null : precisionOverride.getFirst());
    Double minAbs = (precisionOverride == null ? null : precisionOverride.getSecond());

    boolean verboseDebugMode = false;
    if(debugModeRegexes != null){
        for(String regex : debugModeRegexes){
            if(modelName.matches(regex)){
                verboseDebugMode = true;
                break;
            }
        }
    }

    try {
        TFGraphTestAllHelper.checkOnlyOutput(inputs, predictions, modelName, BASE_DIR, MODEL_FILENAME, EXECUTE_WITH, TFGraphTestAllHelper.LOADER, maxRE, minAbs, verboseDebugMode);
        //TFGraphTestAllHelper.checkIntermediate(inputs, modelName, BASE_DIR, MODEL_FILENAME, EXECUTE_WITH, localTestDir);
    } catch (Throwable t){
        log.error("ERROR Executing test: {} - input keys {}", modelName, (inputs == null ? null : inputs.keySet()), t);
        throw t;
    }
    //TFGraphTestAllHelper.checkIntermediate(inputs, modelName, EXECUTE_WITH);
}
 

@Test
public void testDeleteExisting() throws Exception {
    File f = tempDir.newFolder();
    Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData();
    MultiLayerNetwork net = p.getFirst();
    DataSetIterator iter = p.getSecond();


    CheckpointListener l = new CheckpointListener.Builder(f)
            .keepAll()
            .saveEveryNEpochs(1)
            .build();
    net.setListeners(l);

    for(int i=0; i<3; i++ ){
        net.fit(iter);
    }

    //Now, create new listener:
    try{
        l = new CheckpointListener.Builder(f)
                .keepAll()
                .saveEveryNEpochs(1)
                .build();
        fail("Expected exception");
    } catch (IllegalStateException e){
        assertTrue(e.getMessage().contains("Use deleteExisting(true)"));
    }

    l = new CheckpointListener.Builder(f)
            .keepAll()
            .saveEveryNEpochs(1)
            .deleteExisting(true)
            .build();
    net.setListeners(l);

    net.fit(iter);

    File[] fList = f.listFiles();   //checkpoint meta file + 1 checkpoint
    assertNotNull(fList);
    assertEquals(2, fList.length);
}
 

public HyperRect(Pair<float[], float[]> ends) {
    this(ends.getFirst(), ends.getSecond());
}
 

@Test
public void testBufferToIntShapeStrideMethods() {
    //Specifically: Shape.shape(IntBuffer), Shape.shape(DataBuffer)
    //.isRowVectorShape(DataBuffer), .isRowVectorShape(IntBuffer)
    //Shape.size(DataBuffer,int), Shape.size(IntBuffer,int)
    //Also: Shape.stride(IntBuffer), Shape.stride(DataBuffer)
    //Shape.stride(DataBuffer,int), Shape.stride(IntBuffer,int)

    List<List<Pair<INDArray, String>>> lists = new ArrayList<>();
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 4, 12345, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(1, 4, 12345, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 1, 12345, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, new long[]{3, 4, 5}, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, new int[]{3, 4, 5, 6}, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, new int[]{3, 1, 5, 1}, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAll5dTestArraysWithShape(12345, new int[]{3, 4, 5, 6, 7}, DataType.DOUBLE));
    lists.add(NDArrayCreationUtil.getAll6dTestArraysWithShape(12345, new int[]{3, 4, 5, 6, 7, 8}, DataType.DOUBLE));

    val shapes = new long[][] {{3, 4}, {1, 4}, {3, 1}, {3, 4, 5}, {3, 4, 5, 6}, {3, 1, 5, 1}, {3, 4, 5, 6, 7}, {3, 4, 5, 6, 7, 8}};

    for (int i = 0; i < shapes.length; i++) {
        List<Pair<INDArray, String>> list = lists.get(i);
        val shape = shapes[i];

        for (Pair<INDArray, String> p : list) {
            INDArray arr = p.getFirst();

            assertArrayEquals(shape, arr.shape());

            val thisStride = arr.stride();

            val ib = arr.shapeInfo();
            DataBuffer db = arr.shapeInfoDataBuffer();

            //Check shape calculation
            assertEquals(shape.length, Shape.rank(ib));
            assertEquals(shape.length, Shape.rank(db));

            assertArrayEquals(shape, Shape.shape(ib));
            assertArrayEquals(shape, Shape.shape(db));

            for (int j = 0; j < shape.length; j++) {
                assertEquals(shape[j], Shape.size(ib, j));
                assertEquals(shape[j], Shape.size(db, j));

                assertEquals(thisStride[j], Shape.stride(ib, j));
                assertEquals(thisStride[j], Shape.stride(db, j));
            }

            //Check base offset
            assertEquals(Shape.offset(ib), Shape.offset(db));

            //Check offset calculation:
            NdIndexIterator iter = new NdIndexIterator(shape);
            while (iter.hasNext()) {
                val next = iter.next();
                long offset1 = Shape.getOffset(ib, next);

                assertEquals(offset1, Shape.getOffset(db, next));

                switch (shape.length) {
                    case 2:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1]));
                        break;
                    case 3:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2]));
                        break;
                    case 4:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2], next[3]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2], next[3]));
                        break;
                    case 5:
                    case 6:
                        //No 5 and 6d getOffset overloads
                        break;
                    default:
                        throw new RuntimeException();
                }
            }
        }
    }
}
 

@Test
public void testCheckpointListenerEveryTimeUnit() throws Exception {
    File f = tempDir.newFolder();
    Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData();
    MultiLayerNetwork net = p.getFirst();
    DataSetIterator iter = p.getSecond();


    CheckpointListener l = new CheckpointListener.Builder(f)
            .keepLast(3)
            .saveEvery(4900, TimeUnit.MILLISECONDS)
            .build();
    net.setListeners(l);

    for(int i=0; i<3; i++ ){   //10 iterations total
        net.fit(iter);
        Thread.sleep(5000);
    }

    //Expect models saved at iterations: 2, 4, 6, 8 (iterations 0 and 1 shoud happen before first 3 seconds is up)
    //But: keep only 5, 7, 9
    File[] files = f.listFiles();
    Set<Integer> ns = new HashSet<>();
    for(File f2 : files){
        if(!f2.getPath().endsWith(".zip")){
            continue;
        }

        int prefixLength = "checkpoint_".length();
        int num = Integer.parseInt(f2.getName().substring(prefixLength, prefixLength+1));

        MultiLayerNetwork n = ModelSerializer.restoreMultiLayerNetwork(f2, true);
        int expIter = 2 * (num + 1);
        assertEquals(expIter, n.getIterationCount());

        ns.add(n.getIterationCount());
    }

    assertEquals(2, l.availableCheckpoints().size());
    assertEquals(ns.toString(), 2, ns.size());
    System.out.println(ns);
    assertTrue(ns.containsAll(Arrays.asList(2,4)));
}
 

/**
 * Constructor from parsed Keras layer configuration dictionary.
 *
 * @param layerConfig           dictionary containing Keras layer configuration
 * @param enforceTrainingConfig whether to enforce training-related configuration options
 * @param previousLayers        - dictionary containing the previous layers in the topology
 * @throws InvalidKerasConfigurationException     Invalid Keras config
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
public KerasLSTM(Map<String, Object> layerConfig, boolean enforceTrainingConfig,
                 Map<String, ? extends KerasLayer> previousLayers)
        throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
    super(layerConfig, enforceTrainingConfig);

    IWeightInit init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    IWeightInit recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(),
            enforceTrainingConfig, conf, kerasMajorVersion);

    boolean hasBias = getHasBiasFromConfig(layerConfig, conf);

    Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
    this.returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES());

    // TODO: support recurrent dropout
    // double recurrentDropout = KerasRnnUtils.getRecurrentDropout(conf, layerConfig);
    this.unroll = KerasRnnUtils.getUnrollRecurrentLayer(conf, layerConfig);

    LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion);
    LayerConstraint recurrentConstraint = KerasConstraintUtils.getConstraintsFromConfig(
            layerConfig, conf.getLAYER_FIELD_RECURRENT_CONSTRAINT(), conf, kerasMajorVersion);

    Pair<Boolean, Double> maskingConfig = KerasLayerUtils.getMaskingConfiguration(inboundLayerNames, previousLayers);

    LSTM.Builder builder = new LSTM.Builder()
            .gateActivationFunction(getGateActivationFromConfig(layerConfig))
            .forgetGateBiasInit(getForgetBiasInitFromConfig(layerConfig, enforceTrainingConfig))
            .name(this.layerName)
            .nOut(getNOutFromConfig(layerConfig, conf))
            .dropOut(this.dropout)
            .activation(getIActivationFromConfig(layerConfig, conf))
            .weightInit(init)
            .weightInitRecurrent(recurrentInit)
            .biasInit(0.0) // TODO: this is incorrect
            .l1(this.weightL1Regularization)
            .l2(this.weightL2Regularization).dataFormat(RNNFormat.NWC);
    Integer nIn = KerasLayerUtils.getNInFromInputDim(layerConfig, conf);
    if(nIn != null)
        builder.setNIn(nIn);
    if (biasConstraint != null)
        builder.constrainBias(biasConstraint);
    if (weightConstraint != null)
        builder.constrainInputWeights(weightConstraint);
    if (recurrentConstraint != null)
        builder.constrainRecurrent(recurrentConstraint);

    this.layer = builder.build();
    if (!returnSequences) {
        this.layer = new LastTimeStep(this.layer);
    }
    if (maskingConfig.getFirst()) {
        this.layer = new MaskZeroLayer(this.layer, maskingConfig.getSecond());
    }
}
 

private void setShapeInformation(Pair<DataBuffer, long[]> shapeInfo) {
    this.shapeInformation = shapeInfo.getFirst();
    this.jvmShapeInfo = new JvmShapeInfo(shapeInfo.getSecond());
}
 

@Override
public INDArray tensorAlongDimension(long index, int... dimension) {
    if (dimension == null || dimension.length == 0)
        throw new IllegalArgumentException("Invalid input: dimensions not specified (null or length 0)");

    Preconditions.checkArgument(!this.isEmpty(), "tensorAlongDimension(...) can't be used on empty tensors");

    if (dimension.length >= rank()  || dimension.length == 1 && dimension[0] == Integer.MAX_VALUE)
        return this;
    for (int i = 0; i < dimension.length; i++)
        if (dimension[i] < 0)
            dimension[i] += rank();

    //dedup
    if (dimension.length > 1)
        dimension = Ints.toArray(new ArrayList<>(new TreeSet<>(Ints.asList(dimension))));

    if (dimension.length > 1) {
        Arrays.sort(dimension);
    }

    long tads = tensorsAlongDimension(dimension);
    if (index >= tads)
        throw new IllegalArgumentException("Illegal index " + index + " out of tads " + tads);


    if (dimension.length == 1) {
        if (dimension[0] == 0 && isColumnVector()) {
            return this.transpose();
        } else if (dimension[0] == 1 && isRowVector()) {
            return this;
        }
    }

    Pair<DataBuffer, DataBuffer> tadInfo = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(this, dimension);
    DataBuffer shapeInfo = tadInfo.getFirst();
    val jShapeInfo = shapeInfo.asLong();
    val shape = Shape.shape(jShapeInfo);
    val stride = Shape.stride(jShapeInfo);
    long offset = offset() + tadInfo.getSecond().getLong(index);
    val ews = shapeInfo.getLong(jShapeInfo[0] * 2 + 2);
    char tadOrder = (char) shapeInfo.getInt(jShapeInfo[0] * 2 + 3);
    val toTad = Nd4j.create(data(), shape, stride, offset, ews, tadOrder);
    return toTad;
}
 

@Override
public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(true);
    //If this layer is layer L, then epsilon is (w^(L+1)*(d^(L+1))^T) (or equivalent)
    Pair<INDArray, INDArray> zAndPreNorm = preOutputWithPreNorm(true, true, workspaceMgr);
    INDArray z = zAndPreNorm.getFirst(); //Note: using preOutput(INDArray) can't be used as this does a setInput(input) and resets the 'appliedDropout' flag
    INDArray preNorm = zAndPreNorm.getSecond();
    INDArray delta = layerConf().getActivationFn().backprop(z, epsilon).getFirst(); //TODO handle activation function params

    if (maskArray != null) {
        applyMask(delta);
    }

    Gradient ret = new DefaultGradient();

    if(hasBias()){
        INDArray biasGrad = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
        delta.sum(biasGrad, 0); //biasGrad is initialized/zeroed first
        ret.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGrad);
    }

    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr);

    INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{W.size(0), delta.size(0)}, 'f');
    if(hasLayerNorm()) {
        INDArray g = getParam(DefaultParamInitializer.GAIN_KEY);

        INDArray dldg = gradientViews.get(DefaultParamInitializer.GAIN_KEY);
        Nd4j.getExecutioner().exec(new LayerNormBp(preNorm, g, delta, delta, dldg, true, 1));
        ret.gradientForVariable().put(DefaultParamInitializer.GAIN_KEY, dldg);

    }

    epsilonNext = W.mmuli(delta.transpose(),epsilonNext).transpose();   //W.mmul(delta.transpose()).transpose();

    INDArray weightGrad = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY); //f order
    Nd4j.gemm(input.castTo(weightGrad.dataType()), delta, weightGrad, true, false, 1.0, 0.0);           //TODO avoid castTo?
    ret.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGrad);

    weightNoiseParams.clear();

    epsilonNext = backpropDropOutIfPresent(epsilonNext);
    return new Pair<>(ret, epsilonNext);
}