Java Code Examples for org.nd4j.linalg.factory.Nd4j#createBuffer()

@Override
public DataBuffer decompress(DataBuffer buffer) {
    try {

        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();

        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);

        DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements());
        bufferRestored.read(dis);

        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Convert the underlying contents of this {@link Writable}
 * to an nd4j {@link DataBuffer}. Note that this is a *copy*
 * of the underlying buffer.
 * Also note that {@link java.nio.ByteBuffer#allocateDirect(int)}
 * is used for allocation.
 * This should be considered an expensive operation.
 *
 * This buffer should be cached when used. Once used, this can be
 * used in standard Nd4j operations.
 *
 * Beyond that, the reason we have to use allocateDirect
 * is due to nd4j data buffers being stored off heap (whether on cpu or gpu)
 * @param type the type of the data buffer
 * @param elementSize the size of each element in the buffer
 * @return the equivalent nd4j data buffer
 */
public DataBuffer asNd4jBuffer(DataBuffer.Type type,int elementSize) {
    int length = content.length / elementSize;
    DataBuffer ret = Nd4j.createBuffer(ByteBuffer.allocateDirect(content.length),type,length,0);
    for(int i = 0; i < length; i++) {
        switch(type) {
            case DOUBLE:
                ret.put(i,getDouble(i));
                break;
            case INT:
                ret.put(i,getInt(i));
                break;
            case FLOAT:
                ret.put(i,getFloat(i));
               break;
            case LONG:
                ret.put(i,getLong(i));
                break;
        }
    }
    return ret;
}
 

/**
 * Create an ndarray from the specified slices.
 * This will go through and merge all of the
 * data from each slice in to one ndarray
 * which will then take the specified shape
 *
 * @param slices the slices to merge
 * @param shape  the shape of the ndarray
 */
public BaseNDArray(List<INDArray> slices, int[] shape, int[] stride, char ordering) {
    DataBuffer ret = slices.get(0).data().dataType() == (DataBuffer.Type.FLOAT)
            ? Nd4j.createBuffer(new float[ArrayUtil.prod(shape)])
            : Nd4j.createBuffer(new double[ArrayUtil.prod(shape)]);
    this.data = ret;
    setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, 0,
            Shape.elementWiseStride(shape, stride, ordering == 'f'), ordering));
    init(shape, stride);
    //    Shape.setElementWiseStride(this.shapeInfo(),Shape.elementWiseStride(shape, stride, ordering == 'f'));

    if (slices.get(0).isScalar()) {
        for (int i = 0; i < length(); i++) {
            putScalar(i, slices.get(i).getDouble(0));
        }
    } else {
        for (int i = 0; i < slices(); i++) {
            putSlice(i, slices.get(i));
        }
    }
}
 

@Test
public void testDup() throws Exception {
    float[] d1 = new float[] {1, 2, 3, 4};
    DataBuffer d = Nd4j.createBuffer(d1);
    DataBuffer d2 = d.dup();
    assertArrayEquals(d.asFloat(), d2.asFloat(), 0.001f);
}
 

@Test
public void testAssign() {
    DataBuffer assertion = Nd4j.createBuffer(new double[] {1, 2, 3});
    DataBuffer one = Nd4j.createBuffer(new double[] {1});
    DataBuffer twoThree = Nd4j.createBuffer(new double[] {2, 3});
    DataBuffer blank = Nd4j.createBuffer(new double[] {0, 0, 0});
    blank.assign(one, twoThree);
    assertArrayEquals(assertion.asFloat(), blank.asFloat(), 0.0001f);
}
 

@Test
public void testReadWrite() throws Exception {
    DataBuffer assertion = Nd4j.createBuffer(new double[] {1, 2, 3});
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    assertion.write(dos);

    DataBuffer clone = assertion.dup();
    val stream = new DataInputStream(new ByteArrayInputStream(bos.toByteArray()));
    val header = BaseDataBuffer.readHeader(stream);
    assertion.read(stream, header.getLeft(), header.getMiddle(), header.getRight());
    assertArrayEquals(assertion.asFloat(), clone.asFloat(), 0.0001f);
}
 

protected static DataBuffer internalCreateBuffer(float[] data) {
    val perfX = PerformanceTracker.getInstance().helperStartTransaction();

    val buffer = Nd4j.createBuffer(data);
    PerformanceTracker.getInstance().helperRegisterTransaction(0, perfX, data.length * Nd4j.sizeOfDataType(), MemcpyDirection.HOST_TO_HOST);

    return buffer;
}
 

@Test
public void testReallocation() {
    DataBuffer buffer = Nd4j.createBuffer(new double[] {1, 2, 3, 4});
    assertEquals(4, buffer.capacity());
    double[] old = buffer.asDouble();
    buffer.reallocate(6);
    assertEquals(6, buffer.capacity());
    assertArrayEquals(old, buffer.asDouble(), 1e-1);
}
 

@Test
public void testGetSet() throws Exception {
    float[] d1 = new float[] {1, 2, 3, 4};
    DataBuffer d = Nd4j.createBuffer(d1);
    float[] d2 = d.asFloat();
    assertArrayEquals(getFailureMessage(), d1, d2, 1e-1f);

}
 

@Test
public void testBufferCreation() {
    DataBuffer dataBuffer = Nd4j.createBuffer(new double[] {1, 2});
    Pointer pointer = dataBuffer.pointer();
    FloatPointer floatPointer = new FloatPointer(pointer);
    DataBuffer dataBuffer1 = Nd4j.createBuffer(floatPointer, 2);
    assertEquals(2, dataBuffer1.length());
    assertEquals(1.0, dataBuffer1.getDouble(0), 1e-1);
    assertEquals(2.0, dataBuffer1.getDouble(1), 1e-1);
    INDArray arr = Nd4j.create(dataBuffer1);
    System.out.println(arr);
}
 

@Test
public void testPut() throws Exception {
    float[] d1 = new float[] {1, 2, 3, 4};
    DataBuffer d = Nd4j.createBuffer(d1);
    d.put(0, 0.0);
    float[] result = new float[] {0, 2, 3, 4};
    d1 = d.asFloat();
    assertArrayEquals(getFailureMessage(), d1, result, 1e-1f);
}
 

@Override
public INDArray create(float[] data, long[] shape, long[] stride, long offset, char ordering) {
    return new NDArray(Nd4j.createBuffer(data), shape, stride, offset, ordering);
}
 

public BaseNDArray(long[] shape, long[] stride, long offset, char ordering) {
    this(Nd4j.createBuffer(shape.length == 0 ? 1 : ArrayUtil.prodLong(shape)), shape, stride, offset, ordering);
}
 

@Test
public void testBasicSerde1() throws Exception {


    DataBuffer dataBuffer = Nd4j.createBuffer(new int[] {1, 2, 3, 4, 5});
    DataBuffer shapeBuffer = Nd4j.getShapeInfoProvider().createShapeInformation(new int[] {1, 5}).getFirst();
    INDArray intArray = Nd4j.createArrayFromShapeBuffer(dataBuffer, shapeBuffer);

    File tempFile = File.createTempFile("test", "test");
    tempFile.deleteOnExit();

    Nd4j.saveBinary(intArray, tempFile);

    InputStream stream = new FileInputStream(tempFile);
    BufferedInputStream bis = new BufferedInputStream(stream);
    DataInputStream dis = new DataInputStream(bis);

    INDArray loaded = Nd4j.read(dis);

    assertEquals(DataBuffer.Type.INT, loaded.data().dataType());
    assertEquals(DataBuffer.Type.LONG, loaded.shapeInfoDataBuffer().dataType());

    assertEquals(intArray.data().length(), loaded.data().length());

    assertArrayEquals(intArray.data().asInt(), loaded.data().asInt());
}
 

@Override
public DataBuffer decompress(DataBuffer buffer) {
    if (buffer.dataType() != DataBuffer.Type.INT)
        throw new UnsupportedOperationException();

    long compressedLength = buffer.getInt(0);
    long originalLength = buffer.getInt(1);

    DataBuffer result = Nd4j.createBuffer(originalLength);

    CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext();

    PointerPointer extras = new PointerPointer(32).put(1, context.getOldStream());

    //log.info("DEC Source length: {}", buffer.length());
    //log.info("DEC Source: {}", Arrays.toString(buffer.asInt()));

    NativeOpsHolder.getInstance().getDeviceNativeOps().decodeThresholdFloat(extras, AtomicAllocator.getInstance().getPointer(buffer), compressedLength, (FloatPointer) AtomicAllocator.getInstance().getPointer(result));
    AtomicAllocator.getInstance().getAllocationPoint(result).tickDeviceWrite();

    //DataBuffer result = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.THRESHOLD, buffer, getGlobalTypeEx());

    return result;
}
 

@Test
public void testSerialization2() throws Exception {
    DataBuffer bufferOriginal = new CudaFloatDataBuffer(new float[]{1f, 2f, 3f, 4f, 5f});

    DataBuffer bufferHalfs = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.FLOAT, bufferOriginal, DataBuffer.TypeEx.FLOAT16);

    DataTypeUtil.setDTypeForContext(DataBuffer.Type.HALF);

    File tempFile = File.createTempFile("alpha", "11");
    tempFile.deleteOnExit();

    // now we serialize halfs, and we expect it to become floats on other side
    try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(tempFile.getAbsolutePath())))){
        bufferHalfs.write(dos);
    }

    // loading data back from file
    DataInputStream dis = new DataInputStream(new FileInputStream(tempFile.getAbsoluteFile()));

    DataBuffer bufferRestored = Nd4j.createBuffer(bufferOriginal.length());
    bufferRestored.read(dis);

    assertEquals(bufferRestored.dataType(), DataBuffer.Type.HALF);

    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);

    DataBuffer bufferConverted = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.FLOAT16, bufferRestored, DataBuffer.TypeEx.FLOAT);

    assertArrayEquals(bufferOriginal.asFloat(), bufferConverted.asFloat(), 0.01f);
}
 

/**
 * Get the shape from
 * the given int buffer
 * @param buffer the buffer to get the shape information for
 * @return
 */
public static DataBuffer shapeOf(DataBuffer buffer) {
    int rank = (int) buffer.getLong(0);
    return Nd4j.createBuffer(buffer, 1, rank);
}
 

/**
 *
 * @param buffer
 * @param shape
 * @param offset
 */
public BaseNDArray(DataBuffer buffer, int[] shape, long offset) {
    this(Nd4j.createBuffer(buffer, offset, ArrayUtil.prodLong(shape)), shape, Nd4j.getStrides(shape), offset,
            Nd4j.order());
}
 

/**
 * Construct an ndarray of the specified shape
 * with an empty data array
 *
 * @param shape    the shape of the ndarray
 * @param stride   the stride of the ndarray
 * @param offset   the desired offset
 * @param ordering the ordering of the ndarray
 */
public BaseNDArray(int[] shape, int[] stride, long offset, char ordering) {
    this(Nd4j.createBuffer(shape.length == 0 ? 1 : ArrayUtil.prodLong(shape)), shape, stride, offset, ordering);
}
 

/**
 *
 * @param shape
 * @param offset
 * @param ordering
 */
public BaseComplexNDArray(int[] shape, long offset, char ordering) {
    this(Nd4j.createBuffer(ArrayUtil.prodLong(shape) * 2), shape, Nd4j.getComplexStrides(shape, ordering), offset,
                    ordering);
}