Java Code Examples for org.nd4j.common.primitives.Pair#getFirst()
The following examples show how to use
org.nd4j.common.primitives.Pair#getFirst() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BaseOutputLayer.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Override public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) { assertInputSet(true); Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1) INDArray delta = pair.getSecond(); INDArray w = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr); INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{w.size(0), delta.size(0)}, 'f'); epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose(); //Normally we would clear weightNoiseParams here - but we want to reuse them for forward + backward + score // So this is instead done in MultiLayerNetwork/CompGraph backprop methods epsilonNext = backpropDropOutIfPresent(epsilonNext); return new Pair<>(pair.getFirst(), epsilonNext); }
Example 2
Source File: Word2VecPerformerVoid.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Override public void call(Pair<List<VocabWord>, AtomicLong> pair) throws Exception { double numWordsSoFar = wordCount.getValue().doubleValue(); List<VocabWord> sentence = pair.getFirst(); double alpha2 = Math.max(minAlpha, alpha * (1 - (1.0 * numWordsSoFar / (double) totalWords))); int totalNewWords = 0; trainSentence(sentence, alpha2); totalNewWords += sentence.size(); double newWords = totalNewWords + numWordsSoFar; double diff = Math.abs(newWords - lastChecked); if (diff >= 10000) { lastChecked = (int) newWords; log.info("Words so far " + newWords + " out of " + totalWords); } pair.getSecond().getAndAdd((long) totalNewWords); }
Example 3
Source File: TestInvertMatrices.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Test public void testInverseComparison() { List<Pair<INDArray, String>> list = NDArrayCreationUtil.getAllTestMatricesWithShape(10, 10, 12345, DataType.DOUBLE); for (Pair<INDArray, String> p : list) { INDArray orig = p.getFirst(); orig.assign(Nd4j.rand(orig.shape())); INDArray inverse = InvertMatrix.invert(orig, false); RealMatrix rm = CheckUtil.convertToApacheMatrix(orig); RealMatrix rmInverse = new LUDecomposition(rm).getSolver().getInverse(); INDArray expected = CheckUtil.convertFromApacheMatrix(rmInverse, orig.dataType()); assertTrue(p.getSecond(), CheckUtil.checkEntries(expected, inverse, 1e-3, 1e-4)); } }
Example 4
Source File: BertIterator.java From deeplearning4j with Apache License 2.0 | 6 votes |
/** * For use during inference. Will convert a given pair of a list of sentences to features and feature masks as appropriate. * * @param listOnlySentencePairs * @return Pair of INDArrays[], first element is feature arrays and the second is the masks array */ public Pair<INDArray[], INDArray[]> featurizeSentencePairs(List<Pair<String, String>> listOnlySentencePairs) { Preconditions.checkState(sentencePairProvider != null, "The featurizeSentencePairs method is meant for inference with sentence pairs. Use only when the sentence pair provider is set (i.e not null)."); List<Triple<String, String, String>> sentencePairsWithNullLabel = addDummyLabelForPairs(listOnlySentencePairs); SentencePairListProcessed sentencePairListProcessed = tokenizePairsMiniBatch(sentencePairsWithNullLabel); List<Pair<List<String>, String>> tokensAndLabelList = sentencePairListProcessed.getTokensAndLabelList(); int outLength = sentencePairListProcessed.getMaxL(); long[] segIdOnesFrom = sentencePairListProcessed.getSegIdOnesFrom(); if (preProcessor != null) { Pair<INDArray[], INDArray[]> featuresAndMaskArraysPair = convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom); MultiDataSet dummyMDS = new org.nd4j.linalg.dataset.MultiDataSet(featuresAndMaskArraysPair.getFirst(), null, featuresAndMaskArraysPair.getSecond(), null); preProcessor.preProcess(dummyMDS); return new Pair<>(dummyMDS.getFeatures(), dummyMDS.getFeaturesMaskArrays()); } return convertMiniBatchFeatures(tokensAndLabelList, outLength, segIdOnesFrom); }
Example 5
Source File: CenterLossOutputLayer.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Override public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) { assertInputSet(true); Pair<Gradient, INDArray> pair = getGradientsAndDelta(preOutput2d(true, workspaceMgr), workspaceMgr); //Returns Gradient and delta^(this), not Gradient and epsilon^(this-1) INDArray delta = pair.getSecond(); // centers INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY); INDArray l = labels.castTo(centers.dataType()); //Ensure correct dtype (same as params); no-op if already correct dtype INDArray centersForExamples = l.mmul(centers); INDArray dLcdai = input.sub(centersForExamples); INDArray w = getParamWithNoise(CenterLossParamInitializer.WEIGHT_KEY, true, workspaceMgr); INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, w.dataType(), new long[]{w.size(0), delta.size(0)}, 'f'); epsilonNext = w.mmuli(delta.transpose(), epsilonNext).transpose(); double lambda = layerConf().getLambda(); epsilonNext.addi(dLcdai.muli(lambda)); // add center loss here weightNoiseParams.clear(); return new Pair<>(pair.getFirst(), epsilonNext); }
Example 6
Source File: Nd4jTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testExpandDims(){ final List<Pair<INDArray, String>> testMatricesC = NDArrayCreationUtil.getAllTestMatricesWithShape('c', 3, 5, 0xDEAD, DataType.DOUBLE); final List<Pair<INDArray, String>> testMatricesF = NDArrayCreationUtil.getAllTestMatricesWithShape('f', 7, 11, 0xBEEF, DataType.DOUBLE); final ArrayList<Pair<INDArray, String>> testMatrices = new ArrayList<>(testMatricesC); testMatrices.addAll(testMatricesF); for (Pair<INDArray, String> testMatrixPair : testMatrices) { final String recreation = testMatrixPair.getSecond(); final INDArray testMatrix = testMatrixPair.getFirst(); final char ordering = testMatrix.ordering(); val shape = testMatrix.shape(); final int rank = testMatrix.rank(); for (int i = -rank; i <= rank; i++) { final INDArray expanded = Nd4j.expandDims(testMatrix, i); final String message = "Expanding in Dimension " + i + "; Shape before expanding: " + Arrays.toString(shape) + " "+ordering+" Order; Shape after expanding: " + Arrays.toString(expanded.shape()) + " "+expanded.ordering()+"; Input Created via: " + recreation; val tmR = testMatrix.ravel(); val expR = expanded.ravel(); assertEquals(message, 1, expanded.shape()[i < 0 ? i + rank : i]); assertEquals(message, tmR, expR); assertEquals(message, ordering, expanded.ordering()); testMatrix.assign(Nd4j.rand(DataType.DOUBLE, shape)); assertEquals(message, testMatrix.ravel(), expanded.ravel()); } } }
Example 7
Source File: KerasSimpleRnn.java From deeplearning4j with Apache License 2.0 | 4 votes |
/** * Constructor from parsed Keras layer configuration dictionary. * * @param layerConfig dictionary containing Keras layer configuration * @param enforceTrainingConfig whether to enforce training-related configuration options * @param previousLayers dictionary containing the previous layers in the topology * @throws InvalidKerasConfigurationException Invalid Keras config * @throws UnsupportedKerasConfigurationException Unsupported Keras config */ public KerasSimpleRnn(Map<String, Object> layerConfig, boolean enforceTrainingConfig, Map<String, ? extends KerasLayer> previousLayers) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { super(layerConfig, enforceTrainingConfig); IWeightInit init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(), enforceTrainingConfig, conf, kerasMajorVersion); IWeightInit recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(), enforceTrainingConfig, conf, kerasMajorVersion); Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf); this.returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES()); KerasRnnUtils.getRecurrentDropout(conf, layerConfig); this.unroll = KerasRnnUtils.getUnrollRecurrentLayer(conf, layerConfig); Pair<Boolean, Double> maskingConfig = KerasLayerUtils.getMaskingConfiguration(inboundLayerNames, previousLayers); LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion); LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion); LayerConstraint recurrentConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_RECURRENT_CONSTRAINT(), conf, kerasMajorVersion); SimpleRnn.Builder builder = new SimpleRnn.Builder() .name(this.layerName) .nOut(getNOutFromConfig(layerConfig, conf)) .dropOut(this.dropout) .activation(getIActivationFromConfig(layerConfig, conf)) .weightInit(init) .weightInitRecurrent(recurrentInit) .biasInit(0.0) .l1(this.weightL1Regularization) .l2(this.weightL2Regularization).dataFormat(RNNFormat.NWC); Integer nIn = KerasLayerUtils.getNInFromInputDim(layerConfig, conf); if(nIn != null) builder.setNIn(nIn); if (biasConstraint != null) builder.constrainBias(biasConstraint); if (weightConstraint != null) builder.constrainInputWeights(weightConstraint); if (recurrentConstraint != null) builder.constrainRecurrent(recurrentConstraint); this.layer = builder.build(); if (!returnSequences) { this.layer = new LastTimeStep(this.layer); } if (maskingConfig.getFirst()) { this.layer = new MaskZeroLayer(this.layer, maskingConfig.getSecond()); } }
Example 8
Source File: ArbiterStatusListener.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Override public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) { double score; long numParams; int numLayers; String modelConfigJson; int totalNumUpdates; if(candidate instanceof MultiLayerNetwork){ MultiLayerNetwork m = (MultiLayerNetwork)candidate; score = m.score(); numParams = m.numParams(); numLayers = m.getnLayers(); modelConfigJson = m.getLayerWiseConfigurations().toJson(); totalNumUpdates = m.getLayerWiseConfigurations().getIterationCount(); } else if(candidate instanceof ComputationGraph) { ComputationGraph cg = (ComputationGraph)candidate; score = cg.score(); numParams = cg.numParams(); numLayers = cg.getNumLayers(); modelConfigJson = cg.getConfiguration().toJson(); totalNumUpdates = cg.getConfiguration().getIterationCount(); } else { score = 0; numParams = 0; numLayers = 0; totalNumUpdates = 0; modelConfigJson = ""; } int idx = candidateInfo.getIndex(); Pair<IntArrayList, FloatArrayList> pair = candidateScoreVsIter.computeIfAbsent(idx, k -> new Pair<>(new IntArrayList(), new FloatArrayList())); IntArrayList iter = pair.getFirst(); FloatArrayList scores = pair.getSecond(); //Do we need subsampling to avoid having too many data points? int subsamplingFreq = candidateScoreVsIterSubsampleFreq.computeIfAbsent(idx, k -> 1); if(iteration / subsamplingFreq > MAX_SCORE_VS_ITER_PTS){ //Double subsampling frequency and re-parse data subsamplingFreq *= 2; candidateScoreVsIterSubsampleFreq.put(idx, subsamplingFreq); IntArrayList newIter = new IntArrayList(); FloatArrayList newScores = new FloatArrayList(); for( int i=0; i<iter.size(); i++ ){ int it = iter.get(i); if(it % subsamplingFreq == 0){ newIter.add(it); newScores.add(scores.get(i)); } } iter = newIter; scores = newScores; candidateScoreVsIter.put(idx, new Pair<>(iter, scores)); } if(iteration % subsamplingFreq == 0) { iter.add(iteration); scores.add((float) score); } int[] iters = iter.toIntArray(); float[] fScores = new float[iters.length]; for( int i=0; i<iters.length; i++ ){ fScores[i] = scores.get(i); } ModelInfoPersistable p = new ModelInfoPersistable.Builder() .timestamp(candidateInfo.getCreatedTime()) .sessionId(sessionId) .workerId(String.valueOf(candidateInfo.getIndex())) .modelIdx(candidateInfo.getIndex()) .score(candidateInfo.getScore()) .status(candidateInfo.getCandidateStatus()) .scoreVsIter(iters, fScores) .lastUpdateTime(System.currentTimeMillis()) .numParameters(numParams) .numLayers(numLayers) .totalNumUpdates(totalNumUpdates) .paramSpaceValues(candidateInfo.getFlatParams()) .modelConfigJson(modelConfigJson) .exceptionStackTrace(candidateInfo.getExceptionStackTrace()) .build(); lastModelInfoPersistable.put(candidateInfo.getIndex(), p); statsStorage.putUpdate(p); }
Example 9
Source File: GravesLSTMTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
private static void testGravesBackwardBasicHelper(int nIn, int nOut, int lstmNHiddenUnits, int miniBatchSize, int timeSeriesLength) { INDArray inputData = Nd4j.ones(miniBatchSize, nIn, timeSeriesLength); NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder() .layer(new org.deeplearning4j.nn.conf.layers.GravesLSTM.Builder().nIn(nIn) .nOut(lstmNHiddenUnits) .dist(new UniformDistribution(0, 1)).activation(Activation.TANH).build()) .build(); val numParams = conf.getLayer().initializer().numParams(conf); INDArray params = Nd4j.create(1, numParams); GravesLSTM lstm = (GravesLSTM) conf.getLayer().instantiate(conf, null, 0, params, true, params.dataType()); lstm.setBackpropGradientsViewArray(Nd4j.create(1, conf.getLayer().initializer().numParams(conf))); //Set input, do a forward pass: lstm.activate(inputData, false, LayerWorkspaceMgr.noWorkspaces()); assertNotNull(lstm.input()); INDArray epsilon = Nd4j.ones(miniBatchSize, lstmNHiddenUnits, timeSeriesLength); Pair<Gradient, INDArray> out = lstm.backpropGradient(epsilon, LayerWorkspaceMgr.noWorkspaces()); Gradient outGradient = out.getFirst(); INDArray nextEpsilon = out.getSecond(); INDArray biasGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.BIAS_KEY); INDArray inWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.INPUT_WEIGHT_KEY); INDArray recurrentWeightGradient = outGradient.getGradientFor(GravesLSTMParamInitializer.RECURRENT_WEIGHT_KEY); assertNotNull(biasGradient); assertNotNull(inWeightGradient); assertNotNull(recurrentWeightGradient); assertArrayEquals(biasGradient.shape(), new long[] {1, 4 * lstmNHiddenUnits}); assertArrayEquals(inWeightGradient.shape(), new long[] {nIn, 4 * lstmNHiddenUnits}); assertArrayEquals(recurrentWeightGradient.shape(), new long[] {lstmNHiddenUnits, 4 * lstmNHiddenUnits + 3}); assertNotNull(nextEpsilon); assertArrayEquals(nextEpsilon.shape(), new long[] {miniBatchSize, nIn, timeSeriesLength}); //Check update: for (String s : outGradient.gradientForVariable().keySet()) { lstm.update(outGradient.getGradientFor(s), s); } }
Example 10
Source File: TFGraphTestAllHelper.java From deeplearning4j with Apache License 2.0 | 4 votes |
public static void checkIntermediate(Map<String, INDArray> inputs, String modelName, String baseDir, String modelFileName, ExecuteWith execType, BiFunction<File,String,SameDiff> loader, Double maxRelErrorOverride, Double minAbsErrorOverride, File localTestDir, boolean printArraysDebugging) throws IOException { Preconditions.checkArgument((maxRelErrorOverride == null) == (minAbsErrorOverride == null), "Both maxRelErrorOverride and minAbsErrorOverride" + " must be null or both must be provided"); Nd4j.EPS_THRESHOLD = 1e-3; OpExecOrderListener listener = new OpExecOrderListener(); //Used to collect exec order Pair<SameDiff, Map<String,INDArray>> p = getGraphAfterExec(baseDir, modelFileName, modelName, inputs, execType, loader, Collections.singletonList(listener), null, printArraysDebugging); SameDiff graph = p.getFirst(); Map<String,INDArray> sdPredictions = p.getSecond(); //Collect coverage info about ops OpValidation.collectTensorflowImportCoverage(graph); if (!execType.equals(ExecuteWith.JUST_PRINT)) { int count = 0; //Evaluate the nodes in their execution order - this is useful for debugging (as we want the *first* failure // to be detected before later failures) List<String> varNames = new ArrayList<>(); Map<String,SameDiffOp> fns = graph.getOps(); List<String> execOrder = listener.getOpNamesList(); for(String opName : execOrder){ String[] outputs = graph.getOutputsForOp(fns.get(opName).getOp()); Collections.addAll(varNames, outputs); } for (String varName : varNames) { if (!inputs.containsKey(varName)) { //avoiding placeholders INDArray tfValue = intermediateVars(modelName, baseDir, varName, localTestDir); if (tfValue == null) { continue; } log.info("Starting check: variable {}", varName); if (skipNode(modelName, varName)) { log.info("\n\tFORCING no check on " + varName); } else { assertArrayEquals("Shape not equal on node " + varName, tfValue.shape(), graph.getVariable(varName).getShape()); INDArray sdVal = sdPredictions.get(varName); if(maxRelErrorOverride != null){ INDArray diff = Transforms.abs(tfValue.sub(sdVal), false); INDArray absErrorMask = diff.gte(minAbsErrorOverride); //value 1 if x[i] > minAbsError; value 0 otherwise. Used to get rid of 1e-30 vs. 1e-29 type failures INDArray sumAbs = Transforms.abs(tfValue, true).addi(Transforms.abs(sdVal, true)); BooleanIndexing.replaceWhere(sumAbs, 1.0, Conditions.equals(0.0)); //Can only get 0.0 if both are zeros - need to avoid 0/0=NaN INDArray relError = diff.divi(sumAbs); relError.muli(absErrorMask); int countExceeds = Nd4j.getExecutioner().exec(new MatchCondition(relError, Conditions.greaterThan(maxRelErrorOverride))).getInt(0); double maxRE = -1; //Mainly used for analysis in debugger: DifferentialFunction op = null; String[] opInputs = null; if(countExceeds > 0){ maxRE = relError.maxNumber().doubleValue(); //Find the op that this variable is produced by op = graph.getVariableOutputOp(varName); opInputs = graph.getInputsForOp(op); } assertEquals( varName + ": " + countExceeds + " values exceed maxRelError=" + maxRelErrorOverride + " with minAbsError=" + minAbsErrorOverride + "; largest observed relError=" + maxRE, 0, countExceeds); } else { // assertEquals("Value not equal on node " + varName, tfValue, sdVal); if(tfValue.equals(sdVal)){ System.out.println("Pass: " + varName); } else { System.out.println("FAIL: " + varName); System.out.println("TF:\n" + tfValue); System.out.println("SD:\n" + sdVal); } } log.info("Values and shapes equal for {}", varName); count++; } } } assertTrue("No intermediate variables were checked", count > 0); } Nd4j.EPS_THRESHOLD = 1e-5; }
Example 11
Source File: TestComputationGraphNetwork.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test public void testBackwardIrisBasic() { ComputationGraphConfiguration configuration = getIrisGraphConfiguration(); ComputationGraph graph = new ComputationGraph(configuration); graph.init(); MultiLayerConfiguration mlc = getIrisMLNConfiguration(); MultiLayerNetwork net = new MultiLayerNetwork(mlc); net.init(); DataSetIterator iris = new IrisDataSetIterator(150, 150); DataSet ds = iris.next(); //Now: set parameters of both networks to be identical. Then feedforward, and check we get the same outputs Nd4j.getRandom().setSeed(12345); int nParams = (4 * 5 + 5) + (5 * 3 + 3); INDArray params = Nd4j.rand(1, nParams); graph.setParams(params.dup()); net.setParams(params.dup()); INDArray input = ds.getFeatures(); INDArray labels = ds.getLabels(); graph.setInput(0, input.dup()); graph.setLabel(0, labels.dup()); net.setInput(input.dup()); net.setLabels(labels.dup()); //Compute gradients net.computeGradientAndScore(); Pair<Gradient, Double> netGradScore = net.gradientAndScore(); graph.computeGradientAndScore(); Pair<Gradient, Double> graphGradScore = graph.gradientAndScore(); assertEquals(netGradScore.getSecond(), graphGradScore.getSecond(), 1e-3); //Compare gradients Gradient netGrad = netGradScore.getFirst(); Gradient graphGrad = graphGradScore.getFirst(); assertNotNull(graphGrad); assertEquals(netGrad.gradientForVariable().size(), graphGrad.gradientForVariable().size()); assertEquals(netGrad.getGradientFor("0_W"), graphGrad.getGradientFor("firstLayer_W")); assertEquals(netGrad.getGradientFor("0_b"), graphGrad.getGradientFor("firstLayer_b")); assertEquals(netGrad.getGradientFor("1_W"), graphGrad.getGradientFor("outputLayer_W")); assertEquals(netGrad.getGradientFor("1_b"), graphGrad.getGradientFor("outputLayer_b")); }
Example 12
Source File: TFGraphTestAllSameDiff.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test//(timeout = 25000L) public void testOutputOnly() throws Exception { if(TFGraphTestZooModels.isPPC()){ /* Ugly hack to temporarily disable tests on PPC only on CI Issue logged here: https://github.com/deeplearning4j/deeplearning4j/issues/7657 These will be re-enabled for PPC once fixed - in the mean time, remaining tests will be used to detect and prevent regressions */ log.warn("TEMPORARILY SKIPPING TEST ON PPC ARCHITECTURE DUE TO KNOWN JVM CRASH ISSUES - SEE https://github.com/deeplearning4j/deeplearning4j/issues/7657"); OpValidationSuite.ignoreFailing(); } Nd4j.create(1); for(String s : IGNORE_REGEXES){ if(modelName.matches(s)){ log.info("\n\tIGNORE MODEL ON REGEX: {} - regex {}", modelName, s); OpValidationSuite.ignoreFailing(); } } Pair<Double,Double> precisionOverride = TFGraphTestAllHelper.testPrecisionOverride(modelName); Double maxRE = (precisionOverride == null ? null : precisionOverride.getFirst()); Double minAbs = (precisionOverride == null ? null : precisionOverride.getSecond()); boolean verboseDebugMode = false; if(debugModeRegexes != null){ for(String regex : debugModeRegexes){ if(modelName.matches(regex)){ verboseDebugMode = true; break; } } } try { TFGraphTestAllHelper.checkOnlyOutput(inputs, predictions, modelName, BASE_DIR, MODEL_FILENAME, EXECUTE_WITH, TFGraphTestAllHelper.LOADER, maxRE, minAbs, verboseDebugMode); //TFGraphTestAllHelper.checkIntermediate(inputs, modelName, BASE_DIR, MODEL_FILENAME, EXECUTE_WITH, localTestDir); } catch (Throwable t){ log.error("ERROR Executing test: {} - input keys {}", modelName, (inputs == null ? null : inputs.keySet()), t); throw t; } //TFGraphTestAllHelper.checkIntermediate(inputs, modelName, EXECUTE_WITH); }
Example 13
Source File: TestCheckpointListener.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test public void testDeleteExisting() throws Exception { File f = tempDir.newFolder(); Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); CheckpointListener l = new CheckpointListener.Builder(f) .keepAll() .saveEveryNEpochs(1) .build(); net.setListeners(l); for(int i=0; i<3; i++ ){ net.fit(iter); } //Now, create new listener: try{ l = new CheckpointListener.Builder(f) .keepAll() .saveEveryNEpochs(1) .build(); fail("Expected exception"); } catch (IllegalStateException e){ assertTrue(e.getMessage().contains("Use deleteExisting(true)")); } l = new CheckpointListener.Builder(f) .keepAll() .saveEveryNEpochs(1) .deleteExisting(true) .build(); net.setListeners(l); net.fit(iter); File[] fList = f.listFiles(); //checkpoint meta file + 1 checkpoint assertNotNull(fList); assertEquals(2, fList.length); }
Example 14
Source File: HyperRect.java From deeplearning4j with Apache License 2.0 | 4 votes |
public HyperRect(Pair<float[], float[]> ends) { this(ends.getFirst(), ends.getSecond()); }
Example 15
Source File: StaticShapeTests.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test public void testBufferToIntShapeStrideMethods() { //Specifically: Shape.shape(IntBuffer), Shape.shape(DataBuffer) //.isRowVectorShape(DataBuffer), .isRowVectorShape(IntBuffer) //Shape.size(DataBuffer,int), Shape.size(IntBuffer,int) //Also: Shape.stride(IntBuffer), Shape.stride(DataBuffer) //Shape.stride(DataBuffer,int), Shape.stride(IntBuffer,int) List<List<Pair<INDArray, String>>> lists = new ArrayList<>(); lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 4, 12345, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(1, 4, 12345, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 1, 12345, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, new long[]{3, 4, 5}, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, new int[]{3, 4, 5, 6}, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, new int[]{3, 1, 5, 1}, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAll5dTestArraysWithShape(12345, new int[]{3, 4, 5, 6, 7}, DataType.DOUBLE)); lists.add(NDArrayCreationUtil.getAll6dTestArraysWithShape(12345, new int[]{3, 4, 5, 6, 7, 8}, DataType.DOUBLE)); val shapes = new long[][] {{3, 4}, {1, 4}, {3, 1}, {3, 4, 5}, {3, 4, 5, 6}, {3, 1, 5, 1}, {3, 4, 5, 6, 7}, {3, 4, 5, 6, 7, 8}}; for (int i = 0; i < shapes.length; i++) { List<Pair<INDArray, String>> list = lists.get(i); val shape = shapes[i]; for (Pair<INDArray, String> p : list) { INDArray arr = p.getFirst(); assertArrayEquals(shape, arr.shape()); val thisStride = arr.stride(); val ib = arr.shapeInfo(); DataBuffer db = arr.shapeInfoDataBuffer(); //Check shape calculation assertEquals(shape.length, Shape.rank(ib)); assertEquals(shape.length, Shape.rank(db)); assertArrayEquals(shape, Shape.shape(ib)); assertArrayEquals(shape, Shape.shape(db)); for (int j = 0; j < shape.length; j++) { assertEquals(shape[j], Shape.size(ib, j)); assertEquals(shape[j], Shape.size(db, j)); assertEquals(thisStride[j], Shape.stride(ib, j)); assertEquals(thisStride[j], Shape.stride(db, j)); } //Check base offset assertEquals(Shape.offset(ib), Shape.offset(db)); //Check offset calculation: NdIndexIterator iter = new NdIndexIterator(shape); while (iter.hasNext()) { val next = iter.next(); long offset1 = Shape.getOffset(ib, next); assertEquals(offset1, Shape.getOffset(db, next)); switch (shape.length) { case 2: assertEquals(offset1, Shape.getOffset(ib, next[0], next[1])); assertEquals(offset1, Shape.getOffset(db, next[0], next[1])); break; case 3: assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2])); assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2])); break; case 4: assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2], next[3])); assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2], next[3])); break; case 5: case 6: //No 5 and 6d getOffset overloads break; default: throw new RuntimeException(); } } } } }
Example 16
Source File: TestCheckpointListener.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test public void testCheckpointListenerEveryTimeUnit() throws Exception { File f = tempDir.newFolder(); Pair<MultiLayerNetwork, DataSetIterator> p = getNetAndData(); MultiLayerNetwork net = p.getFirst(); DataSetIterator iter = p.getSecond(); CheckpointListener l = new CheckpointListener.Builder(f) .keepLast(3) .saveEvery(4900, TimeUnit.MILLISECONDS) .build(); net.setListeners(l); for(int i=0; i<3; i++ ){ //10 iterations total net.fit(iter); Thread.sleep(5000); } //Expect models saved at iterations: 2, 4, 6, 8 (iterations 0 and 1 shoud happen before first 3 seconds is up) //But: keep only 5, 7, 9 File[] files = f.listFiles(); Set<Integer> ns = new HashSet<>(); for(File f2 : files){ if(!f2.getPath().endsWith(".zip")){ continue; } int prefixLength = "checkpoint_".length(); int num = Integer.parseInt(f2.getName().substring(prefixLength, prefixLength+1)); MultiLayerNetwork n = ModelSerializer.restoreMultiLayerNetwork(f2, true); int expIter = 2 * (num + 1); assertEquals(expIter, n.getIterationCount()); ns.add(n.getIterationCount()); } assertEquals(2, l.availableCheckpoints().size()); assertEquals(ns.toString(), 2, ns.size()); System.out.println(ns); assertTrue(ns.containsAll(Arrays.asList(2,4))); }
Example 17
Source File: KerasLSTM.java From deeplearning4j with Apache License 2.0 | 4 votes |
/** * Constructor from parsed Keras layer configuration dictionary. * * @param layerConfig dictionary containing Keras layer configuration * @param enforceTrainingConfig whether to enforce training-related configuration options * @param previousLayers - dictionary containing the previous layers in the topology * @throws InvalidKerasConfigurationException Invalid Keras config * @throws UnsupportedKerasConfigurationException Unsupported Keras config */ public KerasLSTM(Map<String, Object> layerConfig, boolean enforceTrainingConfig, Map<String, ? extends KerasLayer> previousLayers) throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException { super(layerConfig, enforceTrainingConfig); IWeightInit init = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INIT(), enforceTrainingConfig, conf, kerasMajorVersion); IWeightInit recurrentInit = getWeightInitFromConfig(layerConfig, conf.getLAYER_FIELD_INNER_INIT(), enforceTrainingConfig, conf, kerasMajorVersion); boolean hasBias = getHasBiasFromConfig(layerConfig, conf); Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf); this.returnSequences = (Boolean) innerConfig.get(conf.getLAYER_FIELD_RETURN_SEQUENCES()); // TODO: support recurrent dropout // double recurrentDropout = KerasRnnUtils.getRecurrentDropout(conf, layerConfig); this.unroll = KerasRnnUtils.getUnrollRecurrentLayer(conf, layerConfig); LayerConstraint biasConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_B_CONSTRAINT(), conf, kerasMajorVersion); LayerConstraint weightConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_W_CONSTRAINT(), conf, kerasMajorVersion); LayerConstraint recurrentConstraint = KerasConstraintUtils.getConstraintsFromConfig( layerConfig, conf.getLAYER_FIELD_RECURRENT_CONSTRAINT(), conf, kerasMajorVersion); Pair<Boolean, Double> maskingConfig = KerasLayerUtils.getMaskingConfiguration(inboundLayerNames, previousLayers); LSTM.Builder builder = new LSTM.Builder() .gateActivationFunction(getGateActivationFromConfig(layerConfig)) .forgetGateBiasInit(getForgetBiasInitFromConfig(layerConfig, enforceTrainingConfig)) .name(this.layerName) .nOut(getNOutFromConfig(layerConfig, conf)) .dropOut(this.dropout) .activation(getIActivationFromConfig(layerConfig, conf)) .weightInit(init) .weightInitRecurrent(recurrentInit) .biasInit(0.0) // TODO: this is incorrect .l1(this.weightL1Regularization) .l2(this.weightL2Regularization).dataFormat(RNNFormat.NWC); Integer nIn = KerasLayerUtils.getNInFromInputDim(layerConfig, conf); if(nIn != null) builder.setNIn(nIn); if (biasConstraint != null) builder.constrainBias(biasConstraint); if (weightConstraint != null) builder.constrainInputWeights(weightConstraint); if (recurrentConstraint != null) builder.constrainRecurrent(recurrentConstraint); this.layer = builder.build(); if (!returnSequences) { this.layer = new LastTimeStep(this.layer); } if (maskingConfig.getFirst()) { this.layer = new MaskZeroLayer(this.layer, maskingConfig.getSecond()); } }
Example 18
Source File: BaseNDArray.java From deeplearning4j with Apache License 2.0 | 4 votes |
private void setShapeInformation(Pair<DataBuffer, long[]> shapeInfo) { this.shapeInformation = shapeInfo.getFirst(); this.jvmShapeInfo = new JvmShapeInfo(shapeInfo.getSecond()); }
Example 19
Source File: BaseNDArray.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Override public INDArray tensorAlongDimension(long index, int... dimension) { if (dimension == null || dimension.length == 0) throw new IllegalArgumentException("Invalid input: dimensions not specified (null or length 0)"); Preconditions.checkArgument(!this.isEmpty(), "tensorAlongDimension(...) can't be used on empty tensors"); if (dimension.length >= rank() || dimension.length == 1 && dimension[0] == Integer.MAX_VALUE) return this; for (int i = 0; i < dimension.length; i++) if (dimension[i] < 0) dimension[i] += rank(); //dedup if (dimension.length > 1) dimension = Ints.toArray(new ArrayList<>(new TreeSet<>(Ints.asList(dimension)))); if (dimension.length > 1) { Arrays.sort(dimension); } long tads = tensorsAlongDimension(dimension); if (index >= tads) throw new IllegalArgumentException("Illegal index " + index + " out of tads " + tads); if (dimension.length == 1) { if (dimension[0] == 0 && isColumnVector()) { return this.transpose(); } else if (dimension[0] == 1 && isRowVector()) { return this; } } Pair<DataBuffer, DataBuffer> tadInfo = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(this, dimension); DataBuffer shapeInfo = tadInfo.getFirst(); val jShapeInfo = shapeInfo.asLong(); val shape = Shape.shape(jShapeInfo); val stride = Shape.stride(jShapeInfo); long offset = offset() + tadInfo.getSecond().getLong(index); val ews = shapeInfo.getLong(jShapeInfo[0] * 2 + 2); char tadOrder = (char) shapeInfo.getInt(jShapeInfo[0] * 2 + 3); val toTad = Nd4j.create(data(), shape, stride, offset, ews, tadOrder); return toTad; }
Example 20
Source File: BaseLayer.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Override public Pair<Gradient, INDArray> backpropGradient(INDArray epsilon, LayerWorkspaceMgr workspaceMgr) { assertInputSet(true); //If this layer is layer L, then epsilon is (w^(L+1)*(d^(L+1))^T) (or equivalent) Pair<INDArray, INDArray> zAndPreNorm = preOutputWithPreNorm(true, true, workspaceMgr); INDArray z = zAndPreNorm.getFirst(); //Note: using preOutput(INDArray) can't be used as this does a setInput(input) and resets the 'appliedDropout' flag INDArray preNorm = zAndPreNorm.getSecond(); INDArray delta = layerConf().getActivationFn().backprop(z, epsilon).getFirst(); //TODO handle activation function params if (maskArray != null) { applyMask(delta); } Gradient ret = new DefaultGradient(); if(hasBias()){ INDArray biasGrad = gradientViews.get(DefaultParamInitializer.BIAS_KEY); delta.sum(biasGrad, 0); //biasGrad is initialized/zeroed first ret.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGrad); } INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, true, workspaceMgr); INDArray epsilonNext = workspaceMgr.createUninitialized(ArrayType.ACTIVATION_GRAD, delta.dataType(), new long[]{W.size(0), delta.size(0)}, 'f'); if(hasLayerNorm()) { INDArray g = getParam(DefaultParamInitializer.GAIN_KEY); INDArray dldg = gradientViews.get(DefaultParamInitializer.GAIN_KEY); Nd4j.getExecutioner().exec(new LayerNormBp(preNorm, g, delta, delta, dldg, true, 1)); ret.gradientForVariable().put(DefaultParamInitializer.GAIN_KEY, dldg); } epsilonNext = W.mmuli(delta.transpose(),epsilonNext).transpose(); //W.mmul(delta.transpose()).transpose(); INDArray weightGrad = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY); //f order Nd4j.gemm(input.castTo(weightGrad.dataType()), delta, weightGrad, true, false, 1.0, 0.0); //TODO avoid castTo? ret.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGrad); weightNoiseParams.clear(); epsilonNext = backpropDropOutIfPresent(epsilonNext); return new Pair<>(ret, epsilonNext); }