Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#putScalar()

The following examples show how to use org.nd4j.linalg.api.ndarray.INDArray#putScalar() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YoloGradientCheckTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
private static INDArray yoloLabels(int mb, int c, int h, int w){
    int labelDepth = 4 + c;
    INDArray labels = Nd4j.zeros(mb, labelDepth, h, w);
    //put 1 object per minibatch, at positions (0,0), (1,1) etc.
    //Positions for label boxes: (1,1) to (2,2), (2,2) to (4,4) etc

    for( int i=0; i<mb; i++ ){
        //Class labels
        labels.putScalar(i, 4 + i%c, i%h, i%w, 1);

        //BB coordinates (top left, bottom right)
        labels.putScalar(i, 0, 0, 0, i%w);
        labels.putScalar(i, 1, 0, 0, i%h);
        labels.putScalar(i, 2, 0, 0, (i%w)+1);
        labels.putScalar(i, 3, 0, 0, (i%h)+1);
    }

    return labels;
}
 
Example 2
Source File: CheckUtil.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public static boolean checkMulManually(INDArray first, INDArray second, double maxRelativeDifference,
                double minAbsDifference) {
    //No apache commons element-wise multiply, but can do this manually

    INDArray result = first.mul(second);
    long[] shape = first.shape();

    INDArray expected = Nd4j.zeros(first.shape());

    for (int i = 0; i < shape[0]; i++) {
        for (int j = 0; j < shape[1]; j++) {
            double v = first.getDouble(i, j) * second.getDouble(i, j);
            expected.putScalar(new int[] {i, j}, v);
        }
    }
    if (!checkShape(expected, result))
        return false;
    boolean ok = checkEntries(expected, result, maxRelativeDifference, minAbsDifference);
    if (!ok) {
        INDArray onCopies = Shape.toOffsetZeroCopy(first).mul(Shape.toOffsetZeroCopy(second));
        printFailureDetails(first, second, expected, result, onCopies, "mul");
    }
    return ok;
}
 
Example 3
Source File: DerivativeTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testTanhDerivative() {

    //Derivative of sigmoid: ds(x)/dx = s(x)*(1-s(x))
    //s(x) = 1 / (exp(-x) + 1)
    INDArray z = Nd4j.zeros(100);
    double[] expOut = new double[100];
    for (int i = 0; i < 100; i++) {
        double x = 0.1 * (i - 50);
        z.putScalar(i, x);
        double tanh = FastMath.tanh(x);
        expOut[i] = 1.0 - tanh * tanh;
    }

    INDArray zPrime = Nd4j.getExecutioner().exec(new TanhDerivative(z));

    for (int i = 0; i < 100; i++) {
        double relError = Math.abs(expOut[i] - zPrime.getDouble(i))
                        / (Math.abs(expOut[i]) + Math.abs(zPrime.getDouble(i)));
        assertTrue(relError < REL_ERROR_TOLERANCE);
    }
}
 
Example 4
Source File: TfidfVectorizer.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Override
public INDArray transform(List<String> tokens) {
    INDArray ret = Nd4j.create(1, vocabCache.numWords());

    Map<String, AtomicLong> counts = new HashMap<>();
    for (String token : tokens) {
        if (!counts.containsKey(token))
            counts.put(token, new AtomicLong(0));

        counts.get(token).incrementAndGet();
    }

    for (int i = 0; i < tokens.size(); i++) {
        int idx = vocabCache.indexOf(tokens.get(i));
        if (idx >= 0) {
            double tf_idf = tfidfWord(tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size());
            //log.info("TF-IDF for word: {} -> {} / {} => {}", tokens.get(i), counts.get(tokens.get(i)).longValue(), tokens.size(), tf_idf);
            ret.putScalar(idx, tf_idf);
        }
    }
    return ret;
}
 
Example 5
Source File: BaseNDArrayFactory.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a column vector where each entry is the nth bilinear
 * product of the nth slices of the two tensors.
 */
@Override
public INDArray bilinearProducts(INDArray curr, INDArray in) {
    Preconditions.checkArgument(curr.rank() == 3, "Argument 'curr' must be rank 3. Got input with rank: %s", curr.rank());
    if (in.columns() != 1) {
        throw new AssertionError("Expected a column vector");
    }
    if (in.rows() != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Number of rows in the input does not match number of columns in tensor");
    }
    if (curr.size(curr.shape().length - 2) != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices");
    }

    INDArray ret = Nd4j.create(curr.slices(), 1);
    INDArray inT = in.transpose();
    for (int i = 0; i < curr.slices(); i++) {
        INDArray slice = curr.slice(i);
        INDArray inTTimesSlice = inT.mmul(slice);
        ret.putScalar(i, Nd4j.getBlasWrapper().dot(inTTimesSlice, in));
    }
    return ret;
}
 
Example 6
Source File: ImageUtils.java    From AILibs with GNU Affero General Public License v3.0 5 votes vote down vote up
public static INDArray fastBitmapToMatrix(final FastBitmap bitmap, final ColorSpace colorSpace) {
    INDArray result;

    double[][][] bitmapMatrix;

    switch (colorSpace) {
        case Grayscale:
            result = Nd4j.create(bitmap.toMatrixGrayAsDouble());
            break;
        case ARGB:
            result = Nd4j.create(bitmap.getWidth(), bitmap.getHeight(), 4);
            bitmapMatrix = bitmap.toMatrixRGBAsDouble();

            for (int i = 0; i < bitmap.getWidth(); i++) {
                for (int j = 0; j < bitmap.getHeight(); j++) {
                    result.putScalar(new int[]{i, j, 0}, bitmapMatrix[j][i][0]);
                    result.putScalar(new int[]{i, j, 1}, bitmapMatrix[j][i][1]);
                    result.putScalar(new int[]{i, j, 2}, bitmapMatrix[j][i][2]);
                    result.putScalar(new int[]{i, j, 3}, bitmapMatrix[j][i][3]);
                }
            }
            break;
        case RGB:
            bitmapMatrix = bitmap.toMatrixRGBAsDouble();

            result = Nd4j.create(bitmap.getWidth(), bitmap.getHeight(), 3);
            for (int i = 0; i < bitmap.getWidth(); i++) {
                for (int j = 0; j < bitmap.getHeight(); j++) {
                    result.putScalar(new int[]{i, j, 0}, bitmapMatrix[j][i][0]);
                    result.putScalar(new int[]{i, j, 1}, bitmapMatrix[j][i][1]);
                    result.putScalar(new int[]{i, j, 2}, bitmapMatrix[j][i][2]);
                }
            }
            break;
        default:
            logger.warn("Could not determine color space. Saving an empty matrix...");
            result = Nd4j.create(new double[]{});
    }
    return result;
}
 
Example 7
Source File: ROCTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCompareRocAndRocMultiClass() {
    Nd4j.getRandom().setSeed(12345);

    //For 2 class case: ROC and Multi-class ROC should be the same...
    int nExamples = 200;
    INDArray predictions = Nd4j.rand(nExamples, 2);
    INDArray tempSum = predictions.sum(1);
    predictions.diviColumnVector(tempSum);

    INDArray labels = Nd4j.create(nExamples, 2);
    Random r = new Random(12345);
    for (int i = 0; i < nExamples; i++) {
        labels.putScalar(i, r.nextInt(2), 1.0);
    }

    for (int numSteps : new int[] {30, 0}) { //Steps = 0: exact
        ROC roc = new ROC(numSteps);
        roc.eval(labels, predictions);

        ROCMultiClass rocMultiClass = new ROCMultiClass(numSteps);
        rocMultiClass.eval(labels, predictions);

        double auc = roc.calculateAUC();
        double auc1 = rocMultiClass.calculateAUC(1);

        assertEquals(auc, auc1, 1e-6);
    }
}
 
Example 8
Source File: BaseComplexNDArray.java    From nd4j with Apache License 2.0 5 votes vote down vote up
/**
 * Copy real numbers to arr
 * @param arr the arr to copy to
 */
protected void copyRealTo(INDArray arr) {
    INDArray linear = arr.linearView();
    IComplexNDArray thisLinear = linearView();
    if (arr.isScalar())
        arr.putScalar(0, getReal(0));
    else
        for (int i = 0; i < linear.length(); i++) {
            arr.putScalar(i, thisLinear.getReal(i));
        }

}
 
Example 9
Source File: SVMLightRecordWriterTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testNDArrayWritablesMultilabel() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 0);
    arr3.putScalar(1, 1);
    arr3.putScalar(2, 0);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
            new NDArrayWritable(arr2),
            new IntWritable(2),
            new DoubleWritable(3),
            new NDArrayWritable(arr3),
            new DoubleWritable(1));
    File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0";

    try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}
 
Example 10
Source File: RPUtils.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * Get the search candidates as indices given the input
 * and similarity function
 * @param x the input data to search with
 * @param trees the trees to search
 * @param similarityFunction the function to use for similarity
 * @return the list of indices as the search results
 */
public static INDArray getAllCandidates(INDArray x,List<RPTree> trees,String similarityFunction) {
    List<Integer> candidates = getCandidates(x,trees,similarityFunction);
    Collections.sort(candidates);

    int prevIdx = -1;
    int idxCount = 0;
    List<Pair<Integer,Integer>> scores = new ArrayList<>();
    for(int i = 0; i < candidates.size(); i++) {
        if(candidates.get(i) == prevIdx) {
            idxCount++;
        }
        else if(prevIdx != -1) {
            scores.add(Pair.of(idxCount,prevIdx));
            idxCount = 1;
        }

        prevIdx = i;
    }


    scores.add(Pair.of(idxCount,prevIdx));

    INDArray arr = Nd4j.create(scores.size());
    for(int i = 0; i < scores.size(); i++) {
        arr.putScalar(i,scores.get(i).getSecond());
    }

    return arr;
}
 
Example 11
Source File: CheckUtil.java    From nd4j with Apache License 2.0 5 votes vote down vote up
public static INDArray convertFromApacheMatrix(RealMatrix matrix) {
    int[] shape = new int[] {matrix.getRowDimension(), matrix.getColumnDimension()};
    INDArray out = Nd4j.create(shape);
    for (int i = 0; i < shape[0]; i++) {
        for (int j = 0; j < shape[1]; j++) {
            double value = matrix.getEntry(i, j);
            out.putScalar(new int[] {i, j}, value);
        }
    }
    return out;
}
 
Example 12
Source File: RGBtoGrayscaleDataSetPreProcessorTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void when_colorsAreConverted_expect_grayScaleResult() {
    // Assign
    int numChannels = 3;
    int height = 1;
    int width = 5;

    RGBtoGrayscaleDataSetPreProcessor sut = new RGBtoGrayscaleDataSetPreProcessor();
    INDArray input = Nd4j.create(2, numChannels, height, width);

    // Black, Example 1
    input.putScalar(0, 0, 0, 0, 0.0 );
    input.putScalar(0, 1, 0, 0, 0.0 );
    input.putScalar(0, 2, 0, 0, 0.0 );

    // White, Example 1
    input.putScalar(0, 0, 0, 1, 255.0 );
    input.putScalar(0, 1, 0, 1, 255.0 );
    input.putScalar(0, 2, 0, 1, 255.0 );

    // Red, Example 1
    input.putScalar(0, 0, 0, 2, 255.0 );
    input.putScalar(0, 1, 0, 2, 0.0 );
    input.putScalar(0, 2, 0, 2, 0.0 );

    // Green, Example 1
    input.putScalar(0, 0, 0, 3, 0.0 );
    input.putScalar(0, 1, 0, 3, 255.0 );
    input.putScalar(0, 2, 0, 3, 0.0 );

    // Blue, Example 1
    input.putScalar(0, 0, 0, 4, 0.0 );
    input.putScalar(0, 1, 0, 4, 0.0 );
    input.putScalar(0, 2, 0, 4, 255.0 );


    // Black, Example 2
    input.putScalar(1, 0, 0, 4, 0.0 );
    input.putScalar(1, 1, 0, 4, 0.0 );
    input.putScalar(1, 2, 0, 4, 0.0 );

    // White, Example 2
    input.putScalar(1, 0, 0, 3, 255.0 );
    input.putScalar(1, 1, 0, 3, 255.0 );
    input.putScalar(1, 2, 0, 3, 255.0 );

    // Red, Example 2
    input.putScalar(1, 0, 0, 2, 255.0 );
    input.putScalar(1, 1, 0, 2, 0.0 );
    input.putScalar(1, 2, 0, 2, 0.0 );

    // Green, Example 2
    input.putScalar(1, 0, 0, 1, 0.0 );
    input.putScalar(1, 1, 0, 1, 255.0 );
    input.putScalar(1, 2, 0, 1, 0.0 );

    // Blue, Example 2
    input.putScalar(1, 0, 0, 0, 0.0 );
    input.putScalar(1, 1, 0, 0, 0.0 );
    input.putScalar(1, 2, 0, 0, 255.0 );

    DataSet ds = new DataSet(input, null);

    // Act
    sut.preProcess(ds);

    // Assert
    INDArray result = ds.getFeatures();
    long[] shape = result.shape();

    assertEquals(3, shape.length);
    assertEquals(2, shape[0]);
    assertEquals(1, shape[1]);
    assertEquals(5, shape[2]);

    assertEquals(0.0, result.getDouble(0, 0, 0), 0.05);
    assertEquals(255.0, result.getDouble(0, 0, 1), 0.05);
    assertEquals(255.0 * 0.3, result.getDouble(0, 0, 2), 0.05);
    assertEquals(255.0 * 0.59, result.getDouble(0, 0, 3), 0.05);
    assertEquals(255.0 * 0.11, result.getDouble(0, 0, 4), 0.05);

    assertEquals(0.0, result.getDouble(1, 0, 4), 0.05);
    assertEquals(255.0, result.getDouble(1, 0, 3), 0.05);
    assertEquals(255.0 * 0.3, result.getDouble(1, 0, 2), 0.05);
    assertEquals(255.0 * 0.59, result.getDouble(1, 0, 1), 0.05);
    assertEquals(255.0 * 0.11, result.getDouble(1, 0, 0), 0.05);

}
 
Example 13
Source File: TestMultiModelGradientApplication.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testGradientApplyMultiLayerNetwork() {
    int minibatch = 7;
    int nIn = 10;
    int nOut = 10;

    for (boolean regularization : new boolean[] {false, true}) {
        for (IUpdater u : new IUpdater[] {new Sgd(0.1), new Nesterovs(0.1), new Adam(0.1)}) {

            MultiLayerConfiguration conf =
                            new NeuralNetConfiguration.Builder().seed(12345).activation(Activation.TANH)
                                            .weightInit(WeightInit.XAVIER).updater(u)
                                            .l1(regularization ? 0.2 : 0.0)
                                            .l2(regularization ? 0.3 : 0.0).list()
                                            .layer(0, new DenseLayer.Builder().nIn(nIn).nOut(10).build())
                                            .layer(1, new DenseLayer.Builder().nIn(10).nOut(10).build()).layer(2,
                                                            new OutputLayer.Builder(
                                                                            LossFunctions.LossFunction.MCXENT)
                                                                                            .activation(Activation.SOFTMAX)
                                                                                            .nIn(10).nOut(nOut)
                                                                                            .build())
                                            .build();


            Nd4j.getRandom().setSeed(12345);
            MultiLayerNetwork net1GradCalc = new MultiLayerNetwork(conf);
            net1GradCalc.init();

            Nd4j.getRandom().setSeed(12345);
            MultiLayerNetwork net2GradUpd = new MultiLayerNetwork(conf.clone());
            net2GradUpd.init();

            assertEquals(net1GradCalc.params(), net2GradUpd.params());

            INDArray f = Nd4j.rand(minibatch, nIn);
            INDArray l = Nd4j.create(minibatch, nOut);
            for (int i = 0; i < minibatch; i++) {
                l.putScalar(i, i % nOut, 1.0);
            }
            net1GradCalc.setInput(f);
            net1GradCalc.setLabels(l);

            net2GradUpd.setInput(f);
            net2GradUpd.setLabels(l);

            //Calculate gradient in first net, update and apply it in the second
            //Also: calculate gradient in the second net, just to be sure it isn't modified while doing updating on
            // the other net's gradient
            net1GradCalc.computeGradientAndScore();
            net2GradUpd.computeGradientAndScore();

            Gradient g = net1GradCalc.gradient();
            INDArray gBefore = g.gradient().dup(); //Net 1 gradient should be modified
            INDArray net2GradBefore = net2GradUpd.gradient().gradient().dup(); //But net 2 gradient should not be
            net2GradUpd.getUpdater().update(net2GradUpd, g, 0, 0, minibatch, LayerWorkspaceMgr.noWorkspaces());
            INDArray gAfter = g.gradient().dup();
            INDArray net2GradAfter = net2GradUpd.gradient().gradient().dup();

            assertNotEquals(gBefore, gAfter); //Net 1 gradient should be modified
            assertEquals(net2GradBefore, net2GradAfter); //But net 2 gradient should not be


            //Also: if we apply the gradient using a subi op, we should get the same final params as if we did a fit op
            // on the original network
            net2GradUpd.params().subi(g.gradient());

            net1GradCalc.fit(f, l);
            assertEquals(net1GradCalc.params(), net2GradUpd.params());


            //=============================
            if (!(u instanceof Sgd)) {
                net2GradUpd.getUpdater().getStateViewArray().assign(net1GradCalc.getUpdater().getStateViewArray());
            }
            assertEquals(net1GradCalc.params(), net2GradUpd.params());
            assertEquals(net1GradCalc.getUpdater().getStateViewArray(),
                            net2GradUpd.getUpdater().getStateViewArray());

            //Remove the next 2 lines: fails - as net 1 is 1 iteration ahead
            net1GradCalc.getLayerWiseConfigurations().setIterationCount(0);
            net2GradUpd.getLayerWiseConfigurations().setIterationCount(0);

            for (int i = 0; i < 100; i++) {
                net1GradCalc.fit(f, l);
                net2GradUpd.fit(f, l);
                assertEquals(net1GradCalc.params(), net2GradUpd.params());
            }
        }
    }
}
 
Example 14
Source File: Nd4jTestsComparisonFortran.java    From nd4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testGemmWithOpsCommonsMath() {
    List<Pair<INDArray, String>> first = NDArrayCreationUtil.getAllTestMatricesWithShape(3, 5, SEED);
    List<Pair<INDArray, String>> firstT = NDArrayCreationUtil.getAllTestMatricesWithShape(5, 3, SEED);
    List<Pair<INDArray, String>> second = NDArrayCreationUtil.getAllTestMatricesWithShape(5, 4, SEED);
    List<Pair<INDArray, String>> secondT = NDArrayCreationUtil.getAllTestMatricesWithShape(4, 5, SEED);
    double[] alpha = {1.0, -0.5, 2.5};
    double[] beta = {0.0, -0.25, 1.5};
    INDArray cOrig = Nd4j.create(new int[] {3, 4});
    Random r = new Random(12345);
    for (int i = 0; i < cOrig.size(0); i++) {
        for (int j = 0; j < cOrig.size(1); j++) {
            cOrig.putScalar(new int[] {i, j}, r.nextDouble());
        }
    }

    for (int i = 0; i < first.size(); i++) {
        for (int j = 0; j < second.size(); j++) {
            for (int k = 0; k < alpha.length; k++) {
                for (int m = 0; m < beta.length; m++) {
                    System.out.println((String.format("Running iteration %d %d %d %d", i, j, k, m)));

                    INDArray cff = Nd4j.create(cOrig.shape(), 'f');
                    cff.assign(cOrig);
                    INDArray cft = Nd4j.create(cOrig.shape(), 'f');
                    cft.assign(cOrig);
                    INDArray ctf = Nd4j.create(cOrig.shape(), 'f');
                    ctf.assign(cOrig);
                    INDArray ctt = Nd4j.create(cOrig.shape(), 'f');
                    ctt.assign(cOrig);

                    double a = alpha[k];
                    double b = beta[k];
                    Pair<INDArray, String> p1 = first.get(i);
                    Pair<INDArray, String> p1T = firstT.get(i);
                    Pair<INDArray, String> p2 = second.get(j);
                    Pair<INDArray, String> p2T = secondT.get(j);
                    String errorMsgff = getGemmErrorMsg(i, j, false, false, a, b, p1, p2);
                    String errorMsgft = getGemmErrorMsg(i, j, false, true, a, b, p1, p2T);
                    String errorMsgtf = getGemmErrorMsg(i, j, true, false, a, b, p1T, p2);
                    String errorMsgtt = getGemmErrorMsg(i, j, true, true, a, b, p1T, p2T);

                    assertTrue(errorMsgff, CheckUtil.checkGemm(p1.getFirst(), p2.getFirst(), cff, false, false, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgft, CheckUtil.checkGemm(p1.getFirst(), p2T.getFirst(), cft, false, true, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgtf, CheckUtil.checkGemm(p1T.getFirst(), p2.getFirst(), ctf, true, false, a,
                                    b, 1e-4, 1e-6));
                    assertTrue(errorMsgtt, CheckUtil.checkGemm(p1T.getFirst(), p2T.getFirst(), ctt, true, true, a,
                                    b, 1e-4, 1e-6));
                }
            }
        }
    }
}
 
Example 15
Source File: CNNGradientCheckTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
    public void testCnnWithSpaceToBatch() {
        Nd4j.getRandom().setSeed(12345);
        int nOut = 4;

        int[] minibatchSizes = {2, 4};
        int width = 5;
        int height = 5;
        int inputDepth = 1;

        int[] kernel = {2, 2};
        int[] blocks = {1, 1};

        String[] activations = {"sigmoid", "tanh"};
        SubsamplingLayer.PoolingType[] poolingTypes =
                new SubsamplingLayer.PoolingType[]{SubsamplingLayer.PoolingType.MAX,
                        SubsamplingLayer.PoolingType.AVG, SubsamplingLayer.PoolingType.PNORM};

        for (String afn : activations) {
            for (SubsamplingLayer.PoolingType poolingType : poolingTypes) {
                for (int minibatchSize : minibatchSizes) {
                    INDArray input = Nd4j.rand(minibatchSize, width * height * inputDepth);
                    INDArray labels = Nd4j.zeros(minibatchSize, nOut);
                    for (int i = 0; i < minibatchSize; i++) {
                        labels.putScalar(new int[]{i, i % nOut}, 1.0);
                    }

                    MultiLayerConfiguration conf =
                            new NeuralNetConfiguration.Builder()
                                    .dataType(DataType.DOUBLE)
                                    .updater(new NoOp())
                                    .dist(new NormalDistribution(0, 1))
                                    .list().layer(new ConvolutionLayer.Builder(kernel).nIn(inputDepth)
                                    .cudnnAllowFallback(false)
                                    .nOut(3).build())//output: (5-2+0)/1+1 = 4
                                    .layer(new SpaceToBatchLayer.Builder(blocks).build()) //trivial space to batch
                                    .layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                                            .activation(Activation.SOFTMAX).nIn(4 * 4 * 3)
                                            .nOut(nOut).build())
                                    .setInputType(InputType.convolutionalFlat(height, width, inputDepth))
                                    .build();

                    MultiLayerNetwork net = new MultiLayerNetwork(conf);
                    net.init();

                    String msg = "PoolingType=" + poolingType + ", minibatch=" + minibatchSize + ", activationFn="
                            + afn;

                    if (PRINT_RESULTS) {
                        System.out.println(msg);
//                        for (int j = 0; j < net.getnLayers(); j++)
//                            System.out.println("Layer " + j + " # params: " + net.getLayer(j).numParams());
                    }
                    boolean gradOK = GradientCheckUtil.checkGradients(net, DEFAULT_EPS, DEFAULT_MAX_REL_ERROR,
                            DEFAULT_MIN_ABS_ERROR, PRINT_RESULTS, RETURN_ON_FIRST_FAILURE, input, labels);

                    assertTrue(msg, gradOK);

                    TestUtils.testModelSerialization(net);
                }
            }
        }
    }
 
Example 16
Source File: CNNGradientCheckTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
    public void testSeparableConv2D() {
        int nOut = 2;

        int[] minibatchSizes = new int[]{1, 3};
        int width = 6;
        int height = 6;
        int inputDepth = 3;

        Nd4j.getRandom().setSeed(12345);

        int[] ks = new int[]{1, 3, 3, 1, 3};
        int[] ss = new int[]{1, 1, 1, 2, 2};
        int[] ds = new int[]{1, 1, 2, 2, 2};
        ConvolutionMode[] cms = new ConvolutionMode[]{Truncate, Truncate, Truncate, Truncate, Truncate};
        int[] mb = new int[]{1, 1, 1, 3, 3};

        for (int t = 0; t < ks.length; t++) {

            int k = ks[t];
            int s = ss[t];
            int d = ds[t];
            ConvolutionMode cm = cms[t];
            int minibatchSize = mb[t];

            //Use larger input with larger dilation values (to avoid invalid config)
            int w = d * width;
            int h = d * height;

            INDArray input = Nd4j.rand(minibatchSize, w * h * inputDepth);
            INDArray labels = Nd4j.zeros(minibatchSize, nOut);
            for (int i = 0; i < minibatchSize; i++) {
                labels.putScalar(new int[]{i, i % nOut}, 1.0);
            }

            NeuralNetConfiguration.ListBuilder b = new NeuralNetConfiguration.Builder().seed(12345)
                    .dataType(DataType.DOUBLE)
                    .updater(new NoOp())
                    .activation(Activation.TANH)
                    .convolutionMode(cm)
                    .list()
                    .layer(new SeparableConvolution2D.Builder().name("Separable conv 2D layer")
                            .kernelSize(k, k)
                            .stride(s, s)
                            .dilation(d, d)
                            .depthMultiplier(3)
                            .nIn(inputDepth).nOut(2).build());

            MultiLayerConfiguration conf = b.layer(new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT)
                    .activation(Activation.SOFTMAX).nOut(nOut).build())
                    .setInputType(InputType.convolutionalFlat(h, w, inputDepth)).build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

//            for (int i = 0; i < net.getLayers().length; i++) {
//                System.out.println("nParams, layer " + i + ": " + net.getLayer(i).numParams());
//            }

            String msg = " - mb=" + minibatchSize + ", k="
                    + k + ", s=" + s + ", d=" + d + ", cm=" + cm;
            System.out.println(msg);

            boolean gradOK = GradientCheckUtil.checkGradients(
                    new GradientCheckUtil.MLNConfig().net(net)
                            .input(input).labels(labels)
                            .subset(true).maxPerParam(50));

            assertTrue(msg, gradOK);

            TestUtils.testModelSerialization(net);
        }
    }
 
Example 17
Source File: BNGradientCheckTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
    public void testGradientBNWithCNNandSubsamplingCompGraph() {
        //Parameterized test, testing combinations of:
        // (a) activation function
        // (b) Whether to test at random initialization, or after some learning (i.e., 'characteristic mode of operation')
        // (c) Loss function (with specified output activations)
        // (d) l1 and l2 values
        Activation[] activFns = {Activation.TANH, Activation.IDENTITY};
        boolean doLearningFirst = true;

        LossFunctions.LossFunction[] lossFunctions = {LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD};
        Activation[] outputActivations = {Activation.SOFTMAX}; //i.e., lossFunctions[i] used with outputActivations[i] here

        double[] l2vals = {0.0, 0.1};
        double[] l1vals = {0.0, 0.2}; //i.e., use l2vals[j] with l1vals[j]

        Nd4j.getRandom().setSeed(12345);
        int minibatch = 10;
        int depth = 2;
        int hw = 5;
        int nOut = 3;
        INDArray input = Nd4j.rand(new int[]{minibatch, depth, hw, hw});
        INDArray labels = Nd4j.zeros(minibatch, nOut);
        Random r = new Random(12345);
        for (int i = 0; i < minibatch; i++) {
            labels.putScalar(i, r.nextInt(nOut), 1.0);
        }

        DataSet ds = new DataSet(input, labels);

        for (boolean useLogStd : new boolean[]{true, false}) {
            for (Activation afn : activFns) {
                for (int i = 0; i < lossFunctions.length; i++) {
                    for (int j = 0; j < l2vals.length; j++) {
                        LossFunctions.LossFunction lf = lossFunctions[i];
                        Activation outputActivation = outputActivations[i];

                        ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345)
                                .dataType(DataType.DOUBLE)
                                .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
                                .updater(new NoOp())
                                .dist(new UniformDistribution(-2, 2)).seed(12345L).graphBuilder()
                                .addInputs("in")
                                .addLayer("0", new ConvolutionLayer.Builder(2, 2).stride(1, 1).nOut(3)
                                        .activation(afn).build(), "in")
                                .addLayer("1", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "0")
                                .addLayer("2", new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX)
                                        .kernelSize(2, 2).stride(1, 1).build(), "1")
                                .addLayer("3", new BatchNormalization.Builder().useLogStd(useLogStd).build(), "2")
                                .addLayer("4", new ActivationLayer.Builder().activation(afn).build(), "3")
                                .addLayer("5", new OutputLayer.Builder(lf).activation(outputActivation)
                                        .nOut(nOut).build(), "4")
                                .setOutputs("5").setInputTypes(InputType.convolutional(hw, hw, depth))
                                .build();

                        ComputationGraph net = new ComputationGraph(conf);
                        net.init();
                        String name = new Object() {
                        }.getClass().getEnclosingMethod().getName();

                        if (doLearningFirst) {
                            //Run a number of iterations of learning
                            net.setInput(0, ds.getFeatures());
                            net.setLabels(ds.getLabels());
                            net.computeGradientAndScore();
                            double scoreBefore = net.score();
                            for (int k = 0; k < 20; k++)
                                net.fit(ds);
                            net.computeGradientAndScore();
                            double scoreAfter = net.score();
                            //Can't test in 'characteristic mode of operation' if not learning
                            String msg = name
                                    + " - score did not (sufficiently) decrease during learning - activationFn="
                                    + afn + ", lossFn=" + lf + ", outputActivation=" + outputActivation
                                    + ", doLearningFirst= " + doLearningFirst + " (before=" + scoreBefore
                                    + ", scoreAfter=" + scoreAfter + ")";
                            assertTrue(msg, scoreAfter < 0.9 * scoreBefore);
                        }

                        System.out.println(name + " - activationFn=" + afn + ", lossFn=" + lf
                                + ", outputActivation=" + outputActivation + ", doLearningFirst="
                                + doLearningFirst + ", l1=" + l1vals[j] + ", l2=" + l2vals[j]);
//                        for (int k = 0; k < net.getNumLayers(); k++)
//                            System.out.println("Layer " + k + " # params: " + net.getLayer(k).numParams());

                        //Mean and variance vars are not gradient checkable; mean/variance "gradient" is used to implement running mean/variance calc
                        //i.e., runningMean = decay * runningMean + (1-decay) * batchMean
                        //However, numerical gradient will be 0 as forward pass doesn't depend on this "parameter"
                        Set<String> excludeParams = new HashSet<>(Arrays.asList("1_mean", "1_var", "3_mean", "3_var", "1_log10stdev", "3_log10stdev"));
                        boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.GraphConfig().net(net).inputs(new INDArray[]{input})
                                .labels(new INDArray[]{labels}).excludeParams(excludeParams));

                        assertTrue(gradOK);
                        TestUtils.testModelSerialization(net);
                    }
                }
            }
        }
    }
 
Example 18
Source File: DL4JSequenceRecommender.java    From inception with Apache License 2.0 4 votes vote down vote up
private DataSet vectorize(List<? extends Sample> aData, Object2IntMap<String> aTagset,
        boolean aIncludeLabels)
    throws IOException
{
    // vectorize is pretty fast taking around 1-2ms
    
    // long start = System.currentTimeMillis();
    int maxSentenceLength = traits.getMaxSentenceLength();
    
    // Create data for training
    int embeddingSize = wordVectors.dimensions(); 
    INDArray featureVec = Nd4j.create(aData.size(), embeddingSize, maxSentenceLength);

    // Tags are using a 1-hot encoding
    INDArray labelVec = Nd4j.create(aData.size(), traits.getMaxTagsetSize(), maxSentenceLength);
    
    // Sentences have variable length, so we we need to mask positions not used in short
    // sentences.
    INDArray featureMask = Nd4j.zeros(aData.size(), maxSentenceLength);
    INDArray labelMask = Nd4j.zeros(aData.size(), maxSentenceLength);
    
    // Get word vectors for each word in review, and put them in the training data
    int sampleIdx = 0;
    for (Sample sample : aData) {
        List<String> tokens = sample.getSentence();
        List<String> labels = sample.getTags();
        for (int t = 0; t < Math.min(tokens.size(), maxSentenceLength); t++) {
            String word = tokens.get(t);
            INDArray vector = Nd4j.create(wordVectors.vectorize(word));

            if (vector == null) {
                vector = randUnk;
            }

            featureVec.put(new INDArrayIndex[] { point(sampleIdx), all(), point(t) }, vector);
            featureMask.putScalar(new int[] { sampleIdx, t }, 1.0);

            // exclude padding labels from training
            // compare instances to avoid collision with possible no_label user label
            if (labels != null && labels.get(t) != NO_LABEL) {
                labelMask.putScalar(new int[] { sampleIdx, t }, 1.0);
            }

            if (aIncludeLabels && labels != null) {
                String label = labels.get(t);
                // do not add padding label no_label as predictable label
                if (label != NO_LABEL) {
                    aTagset.computeIfAbsent(label, key -> aTagset.size());
                    labelVec.putScalar(sampleIdx, aTagset.get(label), t, 1.0);
                }
            }
        }
        
        sampleIdx++;
    }

    // log.trace("Vectorizing took {}ms", System.currentTimeMillis() - start);
    
    return new DataSet(featureVec, labelVec, featureMask, labelMask);
}
 
Example 19
Source File: DTypeTests.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testCapsNetDtypes() {
    for (DataType globalDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
        Nd4j.setDefaultDataTypes(globalDtype, globalDtype);
        for (DataType networkDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
            assertEquals(globalDtype, Nd4j.dataType());
            assertEquals(globalDtype, Nd4j.defaultFloatingPointType());

            String msg = "Global dtype: " + globalDtype + ", network dtype: " + networkDtype;

            int primaryCapsDim = 2;
            int primarpCapsChannel = 8;
            int capsule = 5;
            int minibatchSize = 8;
            int routing = 1;
            int capsuleDim = 4;
            int height = 6;
            int width = 6;
            int inputDepth = 4;

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .dataType(networkDtype)
                    .seed(123)
                    .updater(new NoOp())
                    .weightInit(new WeightInitDistribution(new UniformDistribution(-6, 6)))
                    .list()
                    .layer(new PrimaryCapsules.Builder(primaryCapsDim, primarpCapsChannel)
                            .kernelSize(3, 3)
                            .stride(2, 2)
                            .build())
                    .layer(new CapsuleLayer.Builder(capsule, capsuleDim, routing).build())
                    .layer(new CapsuleStrengthLayer.Builder().build())
                    .layer(new ActivationLayer.Builder(new ActivationSoftmax()).build())
                    .layer(new LossLayer.Builder(new LossNegativeLogLikelihood()).build())
                    .setInputType(InputType.convolutional(height, width, inputDepth))
                    .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            INDArray in = Nd4j.rand(networkDtype, minibatchSize, inputDepth * height * width).mul(10)
                    .reshape(-1, inputDepth, height, width);
            INDArray label = Nd4j.zeros(networkDtype, minibatchSize, capsule);
            for (int i = 0; i < minibatchSize; i++) {
                label.putScalar(new int[]{i, i % capsule}, 1.0);
            }

            INDArray out = net.output(in);
            assertEquals(msg, networkDtype, out.dataType());
            List<INDArray> ff = net.feedForward(in);
            for (int i = 0; i < ff.size(); i++) {
                String s = msg + " - layer " + (i - 1) + " - " + (i == 0 ? "input" : net.getLayer(i - 1).conf().getLayer().getClass().getSimpleName());
                assertEquals(s, networkDtype, ff.get(i).dataType());
            }

            net.setInput(in);
            net.setLabels(label);
            net.computeGradientAndScore();

            net.fit(new DataSet(in, label));

            logUsedClasses(net);

            //Now, test mismatched dtypes for input/labels:
            for (DataType inputLabelDtype : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
                INDArray in2 = in.castTo(inputLabelDtype);
                INDArray label2 = label.castTo(inputLabelDtype);
                net.output(in2);
                net.setInput(in2);
                net.setLabels(label2);
                net.computeGradientAndScore();

                net.fit(new DataSet(in2, label2));
            }
        }
    }
}
 
Example 20
Source File: AttentionLayerTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testLearnedSelfAttentionLayer_differentMiniBatchSizes() {
    int nIn = 3;
    int nOut = 2;
    int tsLength = 4;
    int layerSize = 4;
    int numQueries = 3;

    Random r = new Random(12345);
    for (boolean inputMask : new boolean[]{false, true}) {
        for (boolean projectInput : new boolean[]{false, true}) {

            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .dataType(DataType.DOUBLE)
                .activation(Activation.TANH)
                .updater(new NoOp())
                .weightInit(WeightInit.XAVIER)
                .list()
                .layer(new LSTM.Builder().nOut(layerSize).build())
                .layer( projectInput ?
                        new LearnedSelfAttentionLayer.Builder().nOut(4).nHeads(2).nQueries(numQueries).projectInput(true).build()
                        : new LearnedSelfAttentionLayer.Builder().nHeads(1).nQueries(numQueries).projectInput(false).build()
                )
                .layer(new GlobalPoolingLayer.Builder().poolingType(PoolingType.MAX).build())
                .layer(new OutputLayer.Builder().nOut(nOut).activation(Activation.SOFTMAX)
                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                .setInputType(InputType.recurrent(nIn))
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();
        for (int mb : new int[]{3, 1}) {
                INDArray in = Nd4j.rand(DataType.DOUBLE, new int[]{mb, nIn, tsLength});
                INDArray labels = TestUtils.randomOneHot(mb, nOut);
                String maskType = (inputMask ? "inputMask" : "none");

                INDArray inMask = null;
                if (inputMask) {
                    inMask = Nd4j.ones(DataType.INT, mb, tsLength);
                    for (int i = 0; i < mb; i++) {
                        int firstMaskedStep = tsLength - 1 - i;
                        if (firstMaskedStep == 0) {
                            firstMaskedStep = tsLength;
                        }
                        for (int j = firstMaskedStep; j < tsLength; j++) {
                            inMask.putScalar(i, j, 0.0);
                        }
                    }
                }

                String name = "testLearnedSelfAttentionLayer() - mb=" + mb + ", tsLength = " + tsLength + ", maskType=" + maskType + ", projectInput = " + projectInput;
                System.out.println("Starting test: " + name);

                boolean gradOK = GradientCheckUtil.checkGradients(new GradientCheckUtil.MLNConfig().net(net).input(in)
                        .labels(labels).inputMask(inMask).subset(true).maxPerParam(100));
                assertTrue(name, gradOK);
            }
        }
    }
}