Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#data()

The following examples show how to use org.nd4j.linalg.api.ndarray.INDArray#data() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LapackTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testQRSquare() {
    INDArray A = Nd4j.create(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);
    INDArray R = Nd4j.create(A.dataType(), A.columns(), A.columns());

    Nd4j.getBlasWrapper().lapack().geqrf(A, R);

    A.mmuli(R);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 2
Source File: LapackTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testCholeskyL() {
    INDArray A = Nd4j.create(new double[] {2, -1, 1, -1, 2, -1, 1, -1, 2,});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, true);

    A.mmuli(A.transpose());
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 3
Source File: NumpyArray.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up
public NumpyArray(INDArray nd4jArray) {
    Nd4j.getAffinityManager().ensureLocation(nd4jArray, AffinityManager.Location.HOST);
    DataBuffer buff = nd4jArray.data();
    address = buff.pointer().address();
    shape = nd4jArray.shape();
    long[] nd4jStrides = nd4jArray.stride();
    strides = new long[nd4jStrides.length];
    int elemSize = buff.getElementSize();
    for (int i = 0; i < strides.length; i++) {
        strides[i] = nd4jStrides[i] * elemSize;
    }
    dtype = nd4jArray.dataType();
    this.nd4jArray = nd4jArray;
    String cacheKey = address + "_" + nd4jArray.length() + "_" + dtype + "_" + ArrayUtils.toString(strides);
    arrayCache.put(cacheKey, nd4jArray);
}
 
Example 4
Source File: LapackTest.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testCholeskyL() {
    INDArray A = Nd4j.create(new double[] {2, -1, 1, -1, 2, -1, 1, -1, 2,});
    A = A.reshape('c', 3, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, true);

    A.mmuli(A.transpose());
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 5
Source File: LapackTest.java    From nd4j with Apache License 2.0 6 votes vote down vote up
@Test
public void testQRRect() {
    INDArray A = Nd4j.create(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
    A = A.reshape('f', 4, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    INDArray R = Nd4j.create(A.columns(), A.columns());
    Nd4j.getBlasWrapper().lapack().geqrf(A, R);

    A.mmuli(R);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 6
Source File: BinarySerde.java    From nd4j with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    //put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    //put shape next
    allocated.put(shapeBuffer);
    //put codec information next
    allocated.put(codecByteBuffer);
    //finally put the data
    allocated.put(buffer);
    if (rewind)
        allocated.rewind();
}
 
Example 7
Source File: SynchronousFlowController.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public CudaContext prepareActionAllWrite(INDArray... operands) {
    CudaContext context = (CudaContext) allocator.getDeviceContext().getContext();
    int cId = allocator.getDeviceId();

    for (INDArray operand : operands) {
        if (operand == null)
            continue;

        Nd4j.getCompressor().autoDecompress(operand);

        AllocationPoint pointData = allocator.getAllocationPoint(operand);
        AllocationPoint pointShape = allocator.getAllocationPoint(operand.shapeInfoDataBuffer());

        pointData.acquireLock();

        if (pointData.getDeviceId() != cId && pointData.getDeviceId() >= 0) {
            DataBuffer buffer = operand.data().originalDataBuffer() == null ? operand.data()
                            : operand.data().originalDataBuffer();
            allocator.getMemoryHandler().relocateObject(buffer);
        }

        if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) {
            ((JCublasNDArray) operand).setShapeInfoDataBuffer(
                            Nd4j.getConstantHandler().relocateConstantSpace(operand.shapeInfoDataBuffer()));
        }

        prepareDelayedMemory(operand);
        allocator.getAllocationPoint(operand).setCurrentContext(context);
    }
    return context;
}
 
Example 8
Source File: LapackTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCholeskyU() {
    INDArray A = Nd4j.create(new double[] {3, -1, 2, -1, 3, -1, 2, -1, 3,});
    A = A.reshape('f', 3, 3);
    INDArray O = Nd4j.create(A.dataType(), A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, false);
    A = A.transpose().mmul(A);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 9
Source File: DefaultRandom.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public INDArray nextFloat(char order, long[] shape) {
    long length = ArrayUtil.prodLong(shape);
    INDArray ret = Nd4j.create(shape, order);

    DataBuffer data = ret.data();
    for (long i = 0; i < length; i++) {
        data.put(i, nextFloat());
    }

    return ret;
}
 
Example 10
Source File: BasicNDArrayCompressor.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * in place decompression of the given
 * ndarray. If the ndarray isn't compressed
 * this will do nothing
 * @param array the array to decompressed
 *              if it is comprssed
 */
public void decompressi(INDArray array) {
    if (array.data().dataType() != DataType.COMPRESSED)
        return;

    val comp = (CompressedDataBuffer) array.data();
    val descriptor = comp.getCompressionDescriptor();


    if (!codecs.containsKey(descriptor.getCompressionAlgorithm()))
        throw new RuntimeException("Non-existent compression algorithm requested: ["
                        + descriptor.getCompressionAlgorithm() + "]");

    codecs.get(descriptor.getCompressionAlgorithm()).decompressi(array);
}
 
Example 11
Source File: AtomicAllocator.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public void tickHostWrite(INDArray array) {
    DataBuffer buffer =
                    array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();

    tickHostWrite(buffer);
}
 
Example 12
Source File: LapackTest.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Test
public void testCholeskyU() {
    INDArray A = Nd4j.create(new double[] {2, -1, 2, -1, 2, -1, 2, -1, 2,});
    A = A.reshape('f', 3, 3);
    INDArray O = Nd4j.create(A.shape());
    Nd4j.copy(A, O);

    Nd4j.getBlasWrapper().lapack().potrf(A, false);
    A = A.transpose().mmul(A);
    O.subi(A);
    DataBuffer db = O.data();
    for (int i = 0; i < db.length(); i++) {
        assertEquals(0, db.getFloat(i), 1e-5);
    }
}
 
Example 13
Source File: SynchronousFlowController.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
@Override
public CudaContext prepareActionAllWrite(INDArray... operands) {
    val context = allocator.getDeviceContext();
    val cId = allocator.getDeviceId();

    for (INDArray operand : operands) {
        if (operand == null || operand.isEmpty())
            continue;

        Nd4j.getCompressor().autoDecompress(operand);

        val pointData = allocator.getAllocationPoint(operand);
        val pointShape = allocator.getAllocationPoint(operand.shapeInfoDataBuffer());


        if (pointData.getDeviceId() != cId && pointData.getDeviceId() >= 0) {
            DataBuffer buffer = operand.data().originalDataBuffer() == null ? operand.data()
                            : operand.data().originalDataBuffer();
            allocator.getMemoryHandler().relocateObject(buffer);
        }

        if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) {
            ((JCublasNDArray) operand).setShapeInfoDataBuffer(
                            Nd4j.getConstantHandler().relocateConstantSpace(operand.shapeInfoDataBuffer()));
        }

        prepareDelayedMemory(operand);
        allocator.getAllocationPoint(operand).setCurrentContext(context);
    }
    return context;
}
 
Example 14
Source File: DefaultRandom.java    From nd4j with Apache License 2.0 5 votes vote down vote up
@Override
public INDArray nextDouble(char order, long[] shape) {
    long length = ArrayUtil.prodLong(shape);
    INDArray ret = Nd4j.create(shape, order);

    DataBuffer data = ret.data();
    for (long i = 0; i < length; i++) {
        data.put(i, nextDouble());
    }

    return ret;
}
 
Example 15
Source File: AtomicAllocator.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 * This method should be called to make sure that data on host side is actualized
 *
 * @param array
 */
@Override
public void synchronizeHostData(INDArray array) {
    if (array.isEmpty() || array.isS())
        return;

    val buffer = array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();
    synchronizeHostData(buffer);
}
 
Example 16
Source File: BasicNDArrayCompressor.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param array
 * @return
 */
public INDArray decompress(INDArray array) {
    if (array.data().dataType() != DataType.COMPRESSED)
        return array;

    CompressedDataBuffer comp = (CompressedDataBuffer) array.data();
    CompressionDescriptor descriptor = comp.getCompressionDescriptor();

    if (!codecs.containsKey(descriptor.getCompressionAlgorithm()))
        throw new RuntimeException("Non-existent compression algorithm requested: ["
                        + descriptor.getCompressionAlgorithm() + "]");

    return codecs.get(descriptor.getCompressionAlgorithm()).decompress(array);
}
 
Example 17
Source File: AtomicAllocator.java    From nd4j with Apache License 2.0 4 votes vote down vote up
@Override
public AllocationPoint getAllocationPoint(INDArray array) {
    DataBuffer buffer =
                    array.data().originalDataBuffer() == null ? array.data() : array.data().originalDataBuffer();
    return getAllocationPoint(buffer);
}
 
Example 18
Source File: DataBufferTests.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
@Test
public void testAsBytes() {
    INDArray orig = Nd4j.linspace(DataType.INT, 0, 10, 1);

    for (DataType dt : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.BFLOAT16,
            DataType.LONG, DataType.INT, DataType.SHORT, DataType.BYTE, DataType.BOOL,
            DataType.UINT64, DataType.UINT32, DataType.UINT16, DataType.UBYTE}) {
        INDArray arr = orig.castTo(dt);

        byte[] b = arr.data().asBytes();        //NOTE: BIG ENDIAN

        if(ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
            //Switch from big endian (as defined by asBytes which uses big endian) to little endian
            int w = dt.width();
            if (w > 1) {
                int len = b.length / w;
                for (int i = 0; i < len; i++) {
                    for (int j = 0; j < w / 2; j++) {
                        byte temp = b[(i + 1) * w - j - 1];
                        b[(i + 1) * w - j - 1] = b[i * w + j];
                        b[i * w + j] = temp;
                    }
                }
            }
        }

        INDArray arr2 = Nd4j.create(dt, arr.shape());
        ByteBuffer bb = arr2.data().pointer().asByteBuffer();
        bb.position(0);
        bb.put(b);

        Nd4j.getAffinityManager().tagLocation(arr2, AffinityManager.Location.HOST);

        assertEquals(arr.toString(), arr2.toString());
        assertEquals(arr, arr2);

        //Sanity check on data buffer getters:
        DataBuffer db = arr.data();
        DataBuffer db2 = arr2.data();
        for( int i=0; i<10; i++ ){
            assertEquals(db.getDouble(i), db2.getDouble(i), 0);
            assertEquals(db.getFloat(i), db2.getFloat(i), 0);
            assertEquals(db.getInt(i), db2.getInt(i), 0);
            assertEquals(db.getLong(i), db2.getLong(i), 0);
            assertEquals(db.getNumber(i), db2.getNumber(i));
        }

        assertArrayEquals(db.getDoublesAt(0, 10), db2.getDoublesAt(0, 10), 0);
        assertArrayEquals(db.getFloatsAt(0, 10), db2.getFloatsAt(0, 10), 0);
        assertArrayEquals(db.getIntsAt(0, 10), db2.getIntsAt(0, 10));
        assertArrayEquals(db.getLongsAt(0, 10), db2.getLongsAt(0, 10));
    }
}
 
Example 19
Source File: PythonObject.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up
public BytePointer toBytePointer() throws PythonException{
        if (Python.isinstance(this, Python.bytesType())){
            PyObject byteArray = PyByteArray_FromObject(nativePythonObject);
            return PyByteArray_AsString(byteArray);

        }
        else if (Python.isinstance(this, Python.bytearrayType())){
            return PyByteArray_AsString(nativePythonObject);
        }
        else if (Python.isinstance(this, Python.memoryviewType())){

//            PyObject np = PyImport_ImportModule("numpy");
//            PyObject array = PyObject_GetAttrString(np, "asarray");
//            PyObject npArr = PyObject_CallObject(array, nativePythonObject); // Doesn't work
            // Invoke interpreter:
            String tempContext = "temp" + UUID.randomUUID().toString().replace('-', '_');
            String originalContext = Python.getCurrentContext();
            Python.setContext(tempContext);
            PythonExecutioner.setVariable("memview", this);
            PythonExecutioner.exec("import numpy as np\narr = np.frombuffer(memview, dtype='int8')");
            INDArray arr = PythonExecutioner.getVariable("arr").toNumpy().getNd4jArray();
            if(arr.data() instanceof BaseDataBuffer){
                ((BaseDataBuffer)arr.data()).syncToPrimary();
            }
            BytePointer ret = new BytePointer(arr.data().pointer());
            Python.setContext(originalContext);
            Python.deleteContext(tempContext);
            return ret;
        } else {
            PyObject ctypes = PyImport_ImportModule("ctypes");
            PyObject cArrType = PyObject_GetAttrString(ctypes, "Array");
            if (PyObject_IsInstance(nativePythonObject, cArrType) != 0){
                PyObject cVoidP = PyObject_GetAttrString(ctypes, "c_void_p");
                PyObject cast = PyObject_GetAttrString(ctypes, "cast");
                PyObject argsTuple = PyTuple_New(2);
                PyTuple_SetItem(argsTuple, 0, nativePythonObject);
                PyTuple_SetItem(argsTuple, 1, cVoidP);
                PyObject voidPtr = PyObject_Call(cast, argsTuple, null);
                PyObject pyAddress = PyObject_GetAttrString(voidPtr, "value");
                long address = PyLong_AsLong(pyAddress);
                long size = PyObject_Size(nativePythonObject);
                Py_DecRef(ctypes);
                Py_DecRef(cArrType);
                Py_DecRef(argsTuple);
                Py_DecRef(voidPtr);
                Py_DecRef(pyAddress);
                Pointer ptr = NativeOpsHolder.getInstance().getDeviceNativeOps().pointerForAddress(address);
                ptr = ptr.limit(size);
                ptr = ptr.capacity(size);
                return new BytePointer(ptr);
            }
            else{
                throw new PythonException("Expected bytes, bytearray, memoryview or ctypesArray. Received " + Python.type(this).toString());
            }
        }
    }
 
Example 20
Source File: ArrowSerde.java    From deeplearning4j with Apache License 2.0 3 votes vote down vote up
/**
 * Create a {@link Buffer}
 * representing the location metadata of the actual data
 * contents for the ndarrays' {@link DataBuffer}
 * @param bufferBuilder the buffer builder in use
 * @param arr the array to add the underlying data for
 * @return the offset added
 */
public static int addDataForArr(FlatBufferBuilder bufferBuilder, INDArray arr) {
    DataBuffer toAdd = arr.isView() ? arr.dup().data() : arr.data();
    int offset = DataBufferStruct.createDataBufferStruct(bufferBuilder,toAdd);
    int ret = Buffer.createBuffer(bufferBuilder,offset,toAdd.length() * toAdd.getElementSize());
    return ret;

}