Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#rows()

The following examples show how to use org.nd4j.linalg.api.ndarray.INDArray#rows() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IrisUtils.java    From nd4j with Apache License 2.0 6 votes vote down vote up
public static List<DataSet> loadIris(int from, int to) throws IOException {
    ClassPathResource resource = new ClassPathResource("/iris.dat", IrisUtils.class.getClassLoader());
    @SuppressWarnings("unchecked")
    List<String> lines = IOUtils.readLines(resource.getInputStream());
    List<DataSet> list = new ArrayList<>();
    INDArray ret = Nd4j.ones(Math.abs(to - from), 4);
    double[][] outcomes = new double[lines.size()][3];
    int putCount = 0;

    for (int i = from; i < to; i++) {
        String line = lines.get(i);
        String[] split = line.split(",");

        addRow(ret, putCount++, split);

        String outcome = split[split.length - 1];
        double[] rowOutcome = new double[3];
        rowOutcome[Integer.parseInt(outcome)] = 1;
        outcomes[i] = rowOutcome;
    }

    for (int i = 0; i < ret.rows(); i++)
        list.add(new DataSet(ret.getRow(i), Nd4j.create(outcomes[from + i])));

    return list;
}
 
Example 2
Source File: UpdaterTest.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testNadam() {
    int rows = 10;
    int cols = 2;

    NadamUpdater grad = new NadamUpdater(new Nadam());
    grad.setStateViewArray(Nd4j.zeros(1, 2 * rows * cols), new long[] {rows, cols}, 'c', true);
    INDArray W = Nd4j.zeros(rows, cols);
    Distribution dist = Nd4j.getDistributions().createNormal(1e-3, 1e-3);
    for (int i = 0; i < W.rows(); i++)
        W.putRow(i, Nd4j.create(dist.sample(W.columns())));

    for (int i = 0; i < 5; i++) {
        //            String learningRates = String.valueOf("\nAdamUpdater\n " + grad.applyUpdater(W, i)).replaceAll(";", "\n");
        //            System.out.println(learningRates);
        W.addi(Nd4j.randn(rows, cols));
    }
}
 
Example 3
Source File: ShufflesTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public boolean compareColumn(INDArray newData) {
            float[] newMap = measureState(newData);

            if (newMap.length != map.length) {
                System.out.println("Different map lengths");
                return false;
            }

            if (Arrays.equals(map, newMap)) {
//                System.out.println("Maps are equal");
                return false;
            }

            for (int x = 0; x < newData.rows(); x++) {
                INDArray column = newData.getColumn(x);
                double val = column.getDouble(0);
                for (int y = 0; y < column.length(); y++) {
                    if (Math.abs(column.getFloat(y) - val) > Nd4j.EPS_THRESHOLD) {
                        System.out.print("Different data in a column: " + column.getFloat(y));
                        return false;
                    }
                }
            }

            return true;
        }
 
Example 4
Source File: ParagraphVectors.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
/**
 * Get top N elements
 *
 * @param vec the vec to extract the top elements from
 * @param N the number of elements to extract
 * @return the indices and the sorted top N elements
 */
private List<Double> getTopN(INDArray vec, int N) {
    BasicModelUtils.ArrayComparator comparator = new BasicModelUtils.ArrayComparator();
    PriorityQueue<Double[]> queue = new PriorityQueue<>(vec.rows(), comparator);

    for (int j = 0; j < vec.length(); j++) {
        final Double[] pair = new Double[] {vec.getDouble(j), (double) j};
        if (queue.size() < N) {
            queue.add(pair);
        } else {
            Double[] head = queue.peek();
            if (comparator.compare(pair, head) > 0) {
                queue.poll();
                queue.add(pair);
            }
        }
    }

    List<Double> lowToHighSimLst = new ArrayList<>();

    while (!queue.isEmpty()) {
        double ind = queue.poll()[1];
        lowToHighSimLst.add(ind);
    }
    return Lists.reverse(lowToHighSimLst);
}
 
Example 5
Source File: BaseLapack.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Override
public INDArray getLFactor(INDArray A) {
    // FIXME: int cast
    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    INDArray L = Nd4j.create(m, n);
    for (int r = 0; r < m; r++) {
        for (int c = 0; c < n; c++) {
            if (r > c && r < m && c < n) {
                L.putScalar(r, c, A.getFloat(r, c));
            } else if (r < c) {
                L.putScalar(r, c, 0.f);
            } else {
                L.putScalar(r, c, 1.f);
            }
        }
    }
    return L;
}
 
Example 6
Source File: UpdaterTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testNesterovs() {
    int rows = 10;
    int cols = 2;


    NesterovsUpdater grad = new NesterovsUpdater(new Nesterovs(0.5, 0.9));
    grad.setStateViewArray(Nd4j.zeros(1, rows * cols), new long[] {rows, cols}, 'c', true);
    INDArray W = Nd4j.zeros(rows, cols);
    Distribution dist = Nd4j.getDistributions().createNormal(1, 1);
    for (int i = 0; i < W.rows(); i++)
        W.putRow(i, Nd4j.create(dist.sample(W.columns())));

    for (int i = 0; i < 5; i++) {
        //            String learningRates = String.valueOf("\nAdagrad\n " + grad.applyUpdater(W, i)).replaceAll(";", "\n");
        //            System.out.println(learningRates);
        W.addi(Nd4j.randn(rows, cols));
    }
}
 
Example 7
Source File: NDArrayTestsFortran.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testBroadCast() {
    INDArray n = Nd4j.linspace(1, 4, 4, DataType.DOUBLE);
    INDArray broadCasted = n.broadcast(5, 4);
    for (int i = 0; i < broadCasted.rows(); i++) {
        assertEquals(n, broadCasted.getRow(i));
    }

    INDArray broadCast2 = broadCasted.getRow(0).broadcast(5, 4);
    assertEquals(broadCasted, broadCast2);


    INDArray columnBroadcast = n.reshape(4,1).broadcast(4, 5);
    for (int i = 0; i < columnBroadcast.columns(); i++) {
        assertEquals(columnBroadcast.getColumn(i), n.reshape(4));
    }

    INDArray fourD = Nd4j.create(1, 2, 1, 1);
    INDArray broadCasted3 = fourD.broadcast(1, 2, 36, 36);
    assertTrue(Arrays.equals(new long[] {1, 2, 36, 36}, broadCasted3.shape()));
}
 
Example 8
Source File: PruningTest.java    From ml-models with Apache License 2.0 5 votes vote down vote up
@Test
public void testRemoveInnerLoopForComparisons() {
    final double[][] doubles = {
            {0, 1, 1, 1},
            {0, 1, 2, 2},
            {0, 1, 3, 2},
            {0, 2, 3, 3},
    };

    final INDArray indArray = Nd4j.create(doubles);
    final double lambda = 0.6;

    final INDArray[] scores = new INDArray[indArray.rows()];
    for (int i = 0; i < indArray.columns(); i++) {
        final INDArray column = indArray.getColumn(i);
        final INDArray zerosToSum = indArray.subColumnVector(column);
        scores[i] = zerosToSum.condi(Conditions.equals(0)).sum(0).divi(indArray.rows());
    }
    final INDArray indScores = Nd4j.vstack(scores);
    System.out.println("scores = \n" + indScores);

    indScores.condi(Conditions.greaterThan(lambda));
    System.out.println("adjacency matrix with self-loops = \n" + indScores);

    // this line should be optional - not sure better with or without
    Nd4j.doAlongDiagonal(indScores, input -> 0);

    System.out.println("adjacency matrix with self loops removed = \n" + indScores);
}
 
Example 9
Source File: FeatureUtil.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an out come vector from the specified inputs
 *
 * @param index       the index of the label
 * @param numOutcomes the number of possible outcomes
 * @return a binary label matrix used for supervised learning
 */
public static INDArray toOutcomeMatrix(int[] index, long numOutcomes) {
    INDArray ret = Nd4j.create(index.length, numOutcomes);
    for (int i = 0; i < ret.rows(); i++) {
        int[] nums = new int[(int) numOutcomes];
        nums[index[i]] = 1;
        ret.putRow(i, NDArrayUtil.toNDArray(nums));
    }

    return ret;
}
 
Example 10
Source File: AtomicAllocatorTest.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testGpuBlas8akaMultipleThreadsMultipleDevicesSameData() throws Exception {

    Random rnd = new Random(42);

    INDArray array1 = Nd4j.create(100, 757);
    INDArray array2 = Nd4j.create(100, 757);
    for (int y = 0; y < array1.rows(); y++) {
        float[] srcArray1 = new float[757];
        float[] srcArray2 = new float[757];

        for (int x = 0; x < srcArray1.length; x++) {
            srcArray1[x] = rnd.nextFloat();
            srcArray2[x] = rnd.nextFloat();
        }

        array1.putRow(y, Nd4j.create(srcArray1));
        array2.putRow(y, Nd4j.create(srcArray2));
    }

    GpuThreadExternalData[] threads = new GpuThreadExternalData[4];
    for (int x =0; x< threads.length; x++) {
        GpuThreadExternalData thread = new GpuThreadExternalData(array1, array2);
        thread.start();
        threads[x] = thread;
    }

    for (int x = 0; x < threads.length; x++) {
        threads[x].join();
    }

    log.info("Finished");
}
 
Example 11
Source File: ShufflesTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
public float[] measureState(INDArray data) {
    float[] result = new float[data.rows()];

    for (int x = 0; x < data.rows(); x++) {
        result[x] = data.getRow(x).getFloat(0);
    }

    return result;
}
 
Example 12
Source File: RecordConverter.java    From DataVec with Apache License 2.0 5 votes vote down vote up
/**
 * Convert a set of records in to a matrix
 * @param matrix the records ot convert
 * @return the matrix for the records
 */
public static List<List<Writable>> toRecords(INDArray matrix) {
    List<List<Writable>> ret = new ArrayList<>();
    for (int i = 0; i < matrix.rows(); i++) {
        ret.add(RecordConverter.toRecord(matrix.getRow(i)));
    }

    return ret;
}
 
Example 13
Source File: Point.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param matrix
 * @return
 */
public static List<Point> toPoints(INDArray matrix) {
    List<Point> arr = new ArrayList<>(matrix.rows());
    for (int i = 0; i < matrix.rows(); i++) {
        arr.add(new Point(matrix.getRow(i)));
    }

    return arr;
}
 
Example 14
Source File: BaseLapack.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public void gesvd(INDArray A, INDArray S, INDArray U, INDArray VT) {
    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    byte jobu = (byte) (U == null ? 'N' : 'A');
    byte jobvt = (byte) (VT == null ? 'N' : 'A');

    INDArray INFO = Nd4j.createArrayFromShapeBuffer(Nd4j.getDataBufferFactory().createInt(1),
                    Nd4j.getShapeInfoProvider().createShapeInformation(new long[] {1, 1}, DataType.INT).getFirst());

    if (A.data().dataType() == DataType.DOUBLE)
        dgesvd(jobu, jobvt, m, n, A, S, U, VT, INFO);
    else if (A.data().dataType() == DataType.FLOAT)
        sgesvd(jobu, jobvt, m, n, A, S, U, VT, INFO);
    else
        throw new UnsupportedOperationException();

    if (INFO.getInt(0) < 0) {
        throw new Error("Parameter #" + INFO.getInt(0) + " to gesvd() was not valid");
    } else if (INFO.getInt(0) > 0) {
        log.warn("The matrix contains singular elements. Check S matrix at row " + INFO.getInt(0));
    }
}
 
Example 15
Source File: BaseLapack.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public void geqrf(INDArray A, INDArray R) {

    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    INDArray INFO = Nd4j.createArrayFromShapeBuffer(Nd4j.getDataBufferFactory().createInt(1),
                    Nd4j.getShapeInfoProvider().createShapeInformation(new long[] {1, 1}, A.dataType()).getFirst());

    if (R.rows() != A.columns() || R.columns() != A.columns()) {
        throw new Error("geqrf: R must be N x N (n = columns in A)");
    }
    if (A.data().dataType() == DataType.DOUBLE) {
        dgeqrf(m, n, A, R, INFO);
    } else if (A.data().dataType() == DataType.FLOAT) {
        sgeqrf(m, n, A, R, INFO);
    } else {
        throw new UnsupportedOperationException();
    }

    if (INFO.getInt(0) < 0) {
        throw new Error("Parameter #" + INFO.getInt(0) + " to getrf() was not valid");
    }
}
 
Example 16
Source File: FeatureUtil.java    From nd4j with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an out come vector from the specified inputs
 *
 * @param index       the index of the label
 * @param numOutcomes the number of possible outcomes
 * @return a binary label matrix used for supervised learning
 */
public static INDArray toOutcomeMatrix(int[] index, long numOutcomes) {
    INDArray ret = Nd4j.create(index.length, numOutcomes);
    for (int i = 0; i < ret.rows(); i++) {
        int[] nums = new int[(int) numOutcomes];
        nums[index[i]] = 1;
        ret.putRow(i, NDArrayUtil.toNDArray(nums));
    }

    return ret;
}
 
Example 17
Source File: CudaBroadcastTests.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Test
public void execBroadcastOpTimed2() throws Exception {
    Nd4j.create(1);
    System.out.println("A ----------------");
    INDArray array = Nd4j.zeros(2048, 1024);
    System.out.println("0 ----------------");
    INDArray arrayRow = Nd4j.ones(1024);

    System.out.println("1 ----------------");

    float sum = (float) array.sumNumber().doubleValue();
    float sum2 = (float) arrayRow.sumNumber().doubleValue();

    System.out.println("2 ----------------");

    long time1 = System.nanoTime();
    for (int x = 0; x < 1000; x++) {
        array.addiRowVector(arrayRow);
    }
    long time2 = System.nanoTime();

    System.out.println("Execution time: " + ((time2 - time1) / 1000));

    for (int x = 0; x < array.rows(); x++) {
        INDArray row = array.getRow(x);
        for (int y = 0; y < array.columns(); y++) {
            assertEquals("Failed on x.y: ["+x+"."+y+"]",1000f, row.getFloat(y), 0.01);
        }
    }
}
 
Example 18
Source File: BasicClassifier.java    From audiveris with GNU Affero General Public License v3.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void train (Collection<Sample> samples)
{
    logger.info("Training on {} samples", samples.size());

    if (samples.isEmpty()) {
        logger.warn("No sample to retrain neural classifier");

        return;
    }

    StopWatch watch = new StopWatch("train");
    watch.start("shuffle");

    // Shuffle the collection of samples
    final List<Sample> newSamples = new ArrayList<>(samples);
    Collections.shuffle(newSamples);

    // Build raw dataset
    watch.start("getRawDataSet");

    final DataSet dataSet = getRawDataSet(newSamples);
    final INDArray features = dataSet.getFeatures();

    // Record mean and standard deviation for every feature
    watch.start("norms");
    norms = new Norms(features.mean(0), features.std(0));
    norms.stds.addi(Nd4j.scalar(Nd4j.EPS_THRESHOLD)); // Safer, to avoid later division by 0
    logger.debug("means:{}", norms.means);
    logger.debug("stds:{}", norms.stds);
    watch.start("normalize");
    normalize(features);

    // Convert features for NeuralNetwork data format
    int rows = features.rows();
    int cols = features.columns();
    logger.info("samples: {}", rows);
    logger.info("features: {}", cols);

    INDArray labels = dataSet.getLabels();
    double[][] inputs = new double[newSamples.size()][];
    double[][] desiredOutputs = new double[newSamples.size()][];
    watch.start("build input & desiredOutputs");

    for (int ig = 0; ig < rows; ig++) {
        INDArray featureRow = features.getRow(ig);
        double[] ins = new double[cols];
        inputs[ig] = ins;

        for (int j = 0; j < cols; j++) {
            ins[j] = featureRow.getDouble(j);
        }

        INDArray labelRow = labels.getRow(ig);
        double[] des = new double[SHAPE_COUNT];
        desiredOutputs[ig] = des;

        for (int j = 0; j < SHAPE_COUNT; j++) {
            des[j] = labelRow.getDouble(j);
        }
    }

    if (constants.printWatch.isSet()) {
        watch.print();
    }

    // Train
    model.train(inputs, desiredOutputs, listener, listener.getIterationPeriod());

    // Store
    store(FILE_NAME);
}
 
Example 19
Source File: LinAlgExceptions.java    From nd4j with Apache License 2.0 4 votes vote down vote up
public static void assertRows(INDArray n, INDArray n2) {
    if (n.rows() != n2.rows())
        throw new IllegalStateException("Mis matched rows: " + n.rows() + " != " + n2.rows());
}
 
Example 20
Source File: NDArrayIndex.java    From nd4j with Apache License 2.0 3 votes vote down vote up
/**
 * Create from a matrix. The rows are the indices
 * The columns are the individual element in each ndarrayindex
 *
 * @param index the matrix to getFloat indices from
 * @return the indices to getFloat
 */
public static INDArrayIndex[] create(INDArray index) {

    if (index.isMatrix()) {

        if (index.rows() > Integer.MAX_VALUE)
            throw new ND4JArraySizeException();

        NDArrayIndex[] ret = new NDArrayIndex[(int) index.rows()];
        for (int i = 0; i < index.rows(); i++) {
            INDArray row = index.getRow(i);
            val nums = new long[(int) index.getRow(i).columns()];
            for (int j = 0; j < row.columns(); j++) {
                nums[j] = (int) row.getFloat(j);
            }

            NDArrayIndex idx = new NDArrayIndex(nums);
            ret[i] = idx;

        }


        return ret;

    } else if (index.isVector()) {
        long[] indices = NDArrayUtil.toLongs(index);
        return new NDArrayIndex[] {new NDArrayIndex(indices)};
    }


    throw new IllegalArgumentException("Passed in ndarray must be a matrix or a vector");

}