Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#length()

public void saveAsFile(String path) throws IOException {
    try (BufferedWriter write = new BufferedWriter(new FileWriter(new File(path)))) {
        for (int i = 0; i < Y.rows(); i++) {
            StringBuilder sb = new StringBuilder();
            INDArray wordVector = Y.getRow(i);
            for (int j = 0; j < wordVector.length(); j++) {
                sb.append(wordVector.getDouble(j));
                if (j < wordVector.length() - 1)
                    sb.append(",");
            }
            sb.append("\n");
            write.write(sb.toString());
        }
        write.flush();
    }
}
 

/**
 * And over the whole ndarray given some condition, with respect to dimensions
 *
 * @param n    the ndarray to test
 * @param condition the condition to test against
 * @return true if all of the elements meet the specified
 * condition false otherwise
 */
public static boolean[] and(final INDArray n, final Condition condition, int... dimension) {
    if (!(condition instanceof BaseCondition))
        throw new UnsupportedOperationException("Only static Conditions are supported");

    MatchCondition op = new MatchCondition(n, condition);
    INDArray arr = Nd4j.getExecutioner().exec(op, dimension);
    boolean[] result = new boolean[(int) arr.length()];

    long tadLength = Shape.getTADLength(n.shape(), dimension);

    for (int i = 0; i < arr.length(); i++) {
        if (arr.getDouble(i) == tadLength)
            result[i] = true;
        else
            result[i] = false;
    }

    return result;
}
 

@Override
public void scatterUpdate(ScatterUpdate.UpdateOp op, @NonNull INDArray array, @NonNull INDArray indices, @NonNull INDArray updates, @NonNull int[] axis) {
    val tadX = tadManager.getTADOnlyShapeInfo(array, axis);
    val tadY = tadManager.getTADOnlyShapeInfo(updates, axis);

    if (tadY.getSecond().length() != indices.length())
        throw new IllegalStateException("Number of updates doesn't match number of indices. Bad dimensions used?");

    loop.scatterUpdate(null, op.ordinal(), (int) indices.length(),
            array.data().addressPointer(), (LongPointer) tadX.getFirst().addressPointer(), (LongPointer) tadX.getSecond().addressPointer(), null, null, null,
            updates.data().addressPointer(), (LongPointer) tadY.getFirst().addressPointer(), (LongPointer) tadY.getSecond().addressPointer(), null, null, null,
            indices.data().addressPointer(), (LongPointer) indices.shapeInfoDataBuffer().addressPointer(), null, null);

    if (loop.lastErrorCode() != 0)
        throw new RuntimeException(loop.lastErrorMessage());
}
 

@Test
public void testGridPerformance1() throws Exception {
    CudaGridExecutioner executioner = new CudaGridExecutioner();

    INDArray arrayX = Nd4j.create(1024);
    INDArray arrayY = Nd4j.create(1024);

    Set opA = new Set(arrayX, arrayY, arrayX, arrayX.length());
    ScalarAdd opB = new ScalarAdd(arrayX, 2f);

    long time1 = System.nanoTime();
    for (int x = 0; x < 1000000; x++) {
        executioner.exec(opA);
        executioner.exec(opB);
    }
    long time2 = System.nanoTime();

    System.out.println("Execution time Meta: " + ((time2 - time1) / 1000000));
}
 

public boolean compareColumn(INDArray newData) {
            float[] newMap = measureState(newData);

            if (newMap.length != map.length) {
                System.out.println("Different map lengths");
                return false;
            }

            if (Arrays.equals(map, newMap)) {
//                System.out.println("Maps are equal");
                return false;
            }

            for (int x = 0; x < newData.rows(); x++) {
                INDArray column = newData.getColumn(x);
                double val = column.getDouble(0);
                for (int y = 0; y < column.length(); y++) {
                    if (Math.abs(column.getFloat(y) - val) > Nd4j.EPS_THRESHOLD) {
                        System.out.print("Different data in a column: " + column.getFloat(y));
                        return false;
                    }
                }
            }

            return true;
        }
 

/**
 * Asserts matrix multiply rules (columns of left == rows of right or rows of left == columns of right)
 *
 * @param nd1 the left ndarray
 * @param nd2 the right ndarray
 */
public static void assertMultiplies(INDArray nd1, INDArray nd2) {
    if (nd1.rank() == 2 && nd2.rank() == 2 && nd1.columns() == nd2.rows()) {
        return;
    }

    // 1D edge case
    if (nd1.rank() == 2 && nd2.rank() == 1 && nd1.columns() == nd2.length())
        return;

    throw new ND4JIllegalStateException("Cannot execute matrix multiplication: " + Arrays.toString(nd1.shape())
                    + "x" + Arrays.toString(nd2.shape())
                    + (nd1.rank() != 2 || nd2.rank() != 2 ? ": inputs are not matrices"
                                    : ": Column of left array " + nd1.columns() + " != rows of right "
                                                    + nd2.rows()));
}
 

/**
 * trsv solves a system of linear equations whose coefficients are in a triangular matrix.
 *
 * @param order
 * @param Uplo
 * @param TransA
 * @param Diag
 * @param A
 * @param X
 */
@Override
public void trsv(char order, char Uplo, char TransA, char Diag, INDArray A, INDArray X) {
    if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
        OpProfiler.getInstance().processBlasCall(false, A, X);

    if (A.length() > Integer.MAX_VALUE || A.size(0) > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    if (X.data().dataType() == DataType.DOUBLE) {
        DefaultOpExecutioner.validateDataType(DataType.DOUBLE, A, X);
        dtrsv(order, Uplo, TransA, Diag, (int) A.length(), A, (int) A.size(0), X, X.stride(-1));
    } else {
        DefaultOpExecutioner.validateDataType(DataType.FLOAT, A, X);
        strsv(order, Uplo, TransA, Diag, (int) A.length(), A, (int) A.size(0), X, X.stride(-1));
    }

    OpExecutionerUtil.checkForAny(X);
}
 

@Override
public void setStateViewArray(INDArray viewArray, long[] gradientShape, char gradientOrder, boolean initialize) {
    if (!viewArray.isRowVector())
        throw new IllegalArgumentException("Invalid input: expect row vector input");
    if (initialize)
        viewArray.assign(0);
    long length = viewArray.length();
    this.m = viewArray.get(NDArrayIndex.point(0), NDArrayIndex.interval(0, length / 2));
    this.v = viewArray.get(NDArrayIndex.point(0), NDArrayIndex.interval(length / 2, length));

    //Reshape to match the expected shape of the input gradient arrays
    this.m = Shape.newShapeNoCopy(this.m, gradientShape, gradientOrder == 'f');
    this.v = Shape.newShapeNoCopy(this.v, gradientShape, gradientOrder == 'f');
    if (m == null || v == null)
        throw new IllegalStateException("Could not correctly reshape gradient view arrays");

    this.gradientReshapeOrder = gradientOrder;
}
 

@Test
public void testSetSeed1() {
    Random random1 = Nd4j.getRandomFactory().getNewRandomInstance(119);
    Random random2 = Nd4j.getRandomFactory().getNewRandomInstance(119);

    INDArray z01 = Nd4j.create(1000);
    INDArray z11 = Nd4j.create(1000);

    UniformDistribution distribution01 = new UniformDistribution(z01, 1.0, 2.0);
    Nd4j.getExecutioner().exec(distribution01, random1);

    UniformDistribution distribution11 = new UniformDistribution(z11, 1.0, 2.0);
    Nd4j.getExecutioner().exec(distribution11, random2);

    random1.setSeed(1999);
    random2.setSeed(1999);

    INDArray z02 = Nd4j.create(100);
    UniformDistribution distribution02 = new UniformDistribution(z02, 1.0, 2.0);
    Nd4j.getExecutioner().exec(distribution02, random1);

    INDArray z12 = Nd4j.create(100);
    UniformDistribution distribution12 = new UniformDistribution(z12, 1.0, 2.0);
    Nd4j.getExecutioner().exec(distribution12, random2);


    for (int x = 0; x < z01.length(); x++) {
        assertEquals("Failed on element: [" + x + "]", z01.getFloat(x), z11.getFloat(x), 0.01f);
    }

    assertEquals(z01, z11);

    for (int x = 0; x < z02.length(); x++) {
        assertEquals("Failed on element: [" + x + "]", z02.getFloat(x), z12.getFloat(x), 0.01f);
    }

    assertEquals(z02, z12);
}
 

@Test
public void testLeadAndTrail() {
    INDArray fourD = Nd4j.create(1, 2, 1, 1);
    assertEquals(2, fourD.length());
    for (int i = 0; i < fourD.length(); i++)
        assertEquals(0.0, fourD.getDouble(i), 1e-1);

}
 

public TruncatedNormalDistribution(@NonNull INDArray z, @NonNull INDArray means, double stddev) {
    super(z, means, z);
    if (z.length() != means.length())
        throw new IllegalStateException("Result length should be equal to provided Means length");

    if (means.elementWiseStride() < 1)
        throw new IllegalStateException("Means array can't have negative EWS");

    this.mean = 0.0;
    this.stddev = stddev;
    this.extraArgs = new Object[] {this.mean, this.stddev};
}
 

/**
 * Checks, whether given INDArrays are valid time series with a given length.
 *
 * @param array
 * @param length
 * @return True, if the array is a valid time series of the given length. False,
 *         otherwise.
 */
public static boolean isTimeSeries(final int length, final INDArray... array) {
	for (INDArray a : array) {
		if (a.rank() != 1 && a.length() == length) {
			return false;
		}
	}
	return true;
}
 

/**
 * The number of parameters for the model
 *
 * @return the number of parameters for the model
 */
@Override
public long numParams() {
    int ret = 0;
    for (INDArray val : params.values())
        ret += val.length();
    return ret;
}
 

@Test
    public void testSymmetricShuffle1() {
        INDArray features = Nd4j.zeros(10, 10);
        INDArray labels = Nd4j.zeros(10, 3);
        for (int x = 0; x < 10; x++) {
            features.getRow(x).assign(x);
            labels.getRow(x).assign(x);
        }
//        System.out.println(features);

        OrderScanner2D scanner = new OrderScanner2D(features);

        assertArrayEquals(new float[] {0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f}, scanner.getMap(), 0.01f);

        List<INDArray> list = new ArrayList<>();
        list.add(features);
        list.add(labels);

        Nd4j.shuffle(list, 1);

//        System.out.println(features);
//        System.out.println();
//        System.out.println(labels);

        ArrayUtil.argMin(new int[] {});

        assertTrue(scanner.compareRow(features));

        for (int x = 0; x < 10; x++) {
            double val = features.getRow(x).getDouble(0);
            INDArray row = labels.getRow(x);

            for (int y = 0; y < row.length(); y++) {
                assertEquals(val, row.getDouble(y), 0.001);
            }
        }
    }
 

@Test
public void testLabelAndPredictionCounts() {

    int minibatch = 50;
    int nClasses = 3;

    INDArray arr = Nd4j.rand(minibatch, nClasses);
    arr.diviColumnVector(arr.sum(1));
    INDArray labels = Nd4j.zeros(minibatch, nClasses);
    Random r = new Random(12345);
    for (int i = 0; i < minibatch; i++) {
        labels.putScalar(i, r.nextInt(nClasses), 1.0);
    }

    EvaluationCalibration ec = new EvaluationCalibration(5, 5);
    ec.eval(labels, arr);

    int[] expLabelCounts = labels.sum(0).data().asInt();
    int[] expPredictionCount = new int[(int) labels.size(1)];
    INDArray argmax = Nd4j.argMax(arr, 1);
    for (int i = 0; i < argmax.length(); i++) {
        expPredictionCount[argmax.getInt(i)]++;
    }

    assertArrayEquals(expLabelCounts, ec.getLabelCountsEachClass());
    assertArrayEquals(expPredictionCount, ec.getPredictionCountsEachClass());
}
 

@Test
public void testSliceAssertion() {
    INDArray arr = Nd4j.linspace(1, 30, 30).reshape(3, 5, 2);
    INDArray firstRow = arr.slice(0).slice(0);
    for (int i = 0; i < firstRow.length(); i++) {
        System.out.println(firstRow.getDouble(i));
    }
    System.out.println(firstRow);
}
 

@Override
    public INDArray operate(@Nonnull final INDArray W_tl)
            throws DimensionMismatchException {
        if (W_tl.rank() != 2 || W_tl.shape()[0] != numTargets || W_tl.shape()[1] != numLatents) {
            throw new DimensionMismatchException(W_tl.length(), numTargets * numLatents);
        }
        long startTimeRFFT = System.nanoTime();
        /* forward rfft */
        final INDArray W_kl = Nd4j.create(fftSize, numLatents);
        IntStream.range(0, numLatents).parallel().forEach(li ->
                W_kl.get(NDArrayIndex.all(), NDArrayIndex.point(li)).assign(
                        Nd4j.create(F_tt.getForwardFFT(W_tl.get(NDArrayIndex.all(), NDArrayIndex.point(li))),
                                new int[]{fftSize, 1})));
        long endTimeRFFT = System.nanoTime();

        /* apply the preconditioner in the Fourier space */
        long startTimePrecond = System.nanoTime();
        final Map<LinearlySpacedIndexBlock, INDArray> W_kl_map = CoverageModelSparkUtils.partitionINDArrayToMap(fourierSpaceBlocks, W_kl);
        final Broadcast<Map<LinearlySpacedIndexBlock, INDArray>> W_kl_bc = ctx.broadcast(W_kl_map);
        final JavaPairRDD<LinearlySpacedIndexBlock, INDArray> preconditionedWRDD = linOpPairRDD
                .mapToPair(p -> {
                    final INDArray W_kl_chuck = W_kl_bc.value().get(p._1);
                    final INDArray linOp_chunk = p._2;
                    final int blockSize = linOp_chunk.shape()[0];
                    final List<INDArray> linOpWList = IntStream.range(0, blockSize).parallel()
                            .mapToObj(k -> CoverageModelEMWorkspaceMathUtils.linsolve(linOp_chunk.get(NDArrayIndex.point(k)),
                                    W_kl_chuck.get(NDArrayIndex.point(k))))
                            .collect(Collectors.toList());
                    return new Tuple2<>(p._1, Nd4j.vstack(linOpWList));
                });
        W_kl.assign(CoverageModelSparkUtils.assembleINDArrayBlocksFromRDD(preconditionedWRDD, 0));
        W_kl_bc.destroy();
//        final JavaPairRDD<LinearlySpacedIndexBlock, INDArray> W_kl_RDD = CoverageModelSparkUtils.rddFromINDArray(W_kl,
//                fourierSpaceBlocks, ctx, true);
//        W_kl.assign(CoverageModelSparkUtils.assembleINDArrayBlocks(linOpPairRDD.join((W_kl_RDD))
//                .mapValues(p -> {
//                    final INDArray linOp = p._1;
//                    final INDArray W = p._2;
//                    final int blockSize = linOp.shape()[0];
//                    final List<INDArray> linOpWList = IntStream.range(0, blockSize).parallel().mapToObj(k ->
//                            CoverageModelEMWorkspaceMathUtils.linsolve(linOp.get(NDArrayIndex.point(k)),
//                                    W.get(NDArrayIndex.point(k))))
//                            .collect(Collectors.toList());
//                    return Nd4j.vstack(linOpWList);
//                }), false));
//        W_kl_RDD.unpersist();
        long endTimePrecond = System.nanoTime();

        /* irfft */
        long startTimeIRFFT = System.nanoTime();
        final INDArray res = Nd4j.create(numTargets, numLatents);
        IntStream.range(0, numLatents).parallel().forEach(li ->
                res.get(NDArrayIndex.all(), NDArrayIndex.point(li)).assign(
                        F_tt.getInverseFFT(W_kl.get(NDArrayIndex.all(), NDArrayIndex.point(li)))));
        long endTimeIRFFT = System.nanoTime();

        logger.debug("Local FFT timing: " + (endTimeRFFT - startTimeRFFT + endTimeIRFFT - startTimeIRFFT)/1000000 + " ms");
        logger.debug("Spark preconditioner application timing: " + (endTimePrecond - startTimePrecond)/1000000 + " ms");

        return res;
    }
 

@Test
public void testSymmetricShuffle1() {
    INDArray features = Nd4j.zeros(10, 10);
    INDArray labels = Nd4j.zeros(10, 3);
    for (int x = 0; x < 10; x++) {
        features.getRow(x).assign(x);
        labels.getRow(x).assign(x);
    }

    System.out.println(features);

    OrderScanner2D scanner = new OrderScanner2D(features);

    assertArrayEquals(new float[]{0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f}, scanner.getMap(), 0.01f);

    System.out.println();

    List<INDArray> list = new ArrayList<>();
    list.add(features);
    list.add(labels);

    Nd4j.shuffle(list, 1);

    System.out.println(features);

    System.out.println();

    System.out.println(labels);

    ArrayUtil.argMin(new int[]{});

    assertTrue(scanner.compareRow(features));

    for (int x = 0; x < 10; x++) {
        double val = features.getRow(x).getDouble(0);
        INDArray row = labels.getRow(x);

        for (int y = 0; y < row.length(); y++ ) {
            assertEquals(val, row.getDouble(y), 0.001);
        }
    }
}
 

@Override
public INDArray sort(INDArray x, boolean descending) {
    if (x.isScalar())
        return x;

    Nd4j.getExecutioner().push();

    CudaContext context = AtomicAllocator.getInstance().getFlowController().prepareAction(x);

    Pointer ptr = AtomicAllocator.getInstance().getHostPointer(x.shapeInfoDataBuffer());

    PointerPointer extraz = new PointerPointer(ptr, // 0
            context.getOldStream(), // 1
            AtomicAllocator.getInstance().getDeviceIdPointer(), // 2
            null, // 3
            context.getBufferReduction(), // 4
            context.getBufferScalar(), // 5
            null, // 6
            ptr, // 7
            AtomicAllocator.getInstance().getHostPointer(x.shapeInfoDataBuffer()), // 8
            ptr, // 9
            ptr, // 10
            ptr, // 11
            ptr, // 12
            ptr, // 13
            ptr, // 14
            ptr, // special pointer for IsMax  // 15
            ptr, // special pointer for IsMax  // 16
            ptr, // special pointer for IsMax // 17
            new CudaPointer(0));

    // we're sending > 10m elements to radixSort
    boolean isRadix = !x.isView() && (x.length() > 1024 * 1024 * 10);
    INDArray tmpX = x;

    // we need to guarantee all threads are finished here
    if (isRadix)
        Nd4j.getExecutioner().commit();


    nativeOps.sort(extraz,
                null,
                (LongPointer) x.shapeInfoDataBuffer().addressPointer(),
                AtomicAllocator.getInstance().getPointer(tmpX, context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tmpX.shapeInfoDataBuffer(), context),
                descending
        );

    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    AtomicAllocator.getInstance().getFlowController().registerAction(context, x);

    return x;
}
 

@Test
public void testGridFlowFlush1() throws Exception {
    CudaGridExecutioner executioner = new CudaGridExecutioner();

    INDArray arrayX = Nd4j.create(10);
    INDArray arrayY = Nd4j.create(new float[] {1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f, 1f});
    INDArray exp = Nd4j.create(new float[] {3f, 3f, 3f, 3f, 3f, 3f, 3f, 3f, 3f, 3f});

    Set opA = new Set(arrayX, arrayY, arrayX, arrayX.length());

    executioner.exec(opA);

    executioner.flushQueue();

    assertEquals(arrayY, arrayX);
}