Java Code Examples for org.nd4j.linalg.factory.Nd4j#createBuffer()
The following examples show how to use
org.nd4j.linalg.factory.Nd4j#createBuffer() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Gzip.java From nd4j with Apache License 2.0 | 6 votes |
@Override public DataBuffer decompress(DataBuffer buffer) { try { CompressedDataBuffer compressed = (CompressedDataBuffer) buffer; CompressionDescriptor descriptor = compressed.getCompressionDescriptor(); BytePointer pointer = (BytePointer) compressed.addressPointer(); ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes()); GZIPInputStream gzip = new GZIPInputStream(bis); DataInputStream dis = new DataInputStream(gzip); DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements()); bufferRestored.read(dis); return bufferRestored; } catch (Exception e) { throw new RuntimeException(e); } }
Example 2
Source File: BytesWritable.java From DataVec with Apache License 2.0 | 6 votes |
/** * Convert the underlying contents of this {@link Writable} * to an nd4j {@link DataBuffer}. Note that this is a *copy* * of the underlying buffer. * Also note that {@link java.nio.ByteBuffer#allocateDirect(int)} * is used for allocation. * This should be considered an expensive operation. * * This buffer should be cached when used. Once used, this can be * used in standard Nd4j operations. * * Beyond that, the reason we have to use allocateDirect * is due to nd4j data buffers being stored off heap (whether on cpu or gpu) * @param type the type of the data buffer * @param elementSize the size of each element in the buffer * @return the equivalent nd4j data buffer */ public DataBuffer asNd4jBuffer(DataBuffer.Type type,int elementSize) { int length = content.length / elementSize; DataBuffer ret = Nd4j.createBuffer(ByteBuffer.allocateDirect(content.length),type,length,0); for(int i = 0; i < length; i++) { switch(type) { case DOUBLE: ret.put(i,getDouble(i)); break; case INT: ret.put(i,getInt(i)); break; case FLOAT: ret.put(i,getFloat(i)); break; case LONG: ret.put(i,getLong(i)); break; } } return ret; }
Example 3
Source File: BaseNDArray.java From nd4j with Apache License 2.0 | 6 votes |
/** * Create an ndarray from the specified slices. * This will go through and merge all of the * data from each slice in to one ndarray * which will then take the specified shape * * @param slices the slices to merge * @param shape the shape of the ndarray */ public BaseNDArray(List<INDArray> slices, int[] shape, int[] stride, char ordering) { DataBuffer ret = slices.get(0).data().dataType() == (DataBuffer.Type.FLOAT) ? Nd4j.createBuffer(new float[ArrayUtil.prod(shape)]) : Nd4j.createBuffer(new double[ArrayUtil.prod(shape)]); this.data = ret; setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, 0, Shape.elementWiseStride(shape, stride, ordering == 'f'), ordering)); init(shape, stride); // Shape.setElementWiseStride(this.shapeInfo(),Shape.elementWiseStride(shape, stride, ordering == 'f')); if (slices.get(0).isScalar()) { for (int i = 0; i < length(); i++) { putScalar(i, slices.get(i).getDouble(0)); } } else { for (int i = 0; i < slices(); i++) { putSlice(i, slices.get(i)); } } }
Example 4
Source File: FloatDataBufferTest.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testDup() throws Exception { float[] d1 = new float[] {1, 2, 3, 4}; DataBuffer d = Nd4j.createBuffer(d1); DataBuffer d2 = d.dup(); assertArrayEquals(d.asFloat(), d2.asFloat(), 0.001f); }
Example 5
Source File: FloatDataBufferTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testAssign() { DataBuffer assertion = Nd4j.createBuffer(new double[] {1, 2, 3}); DataBuffer one = Nd4j.createBuffer(new double[] {1}); DataBuffer twoThree = Nd4j.createBuffer(new double[] {2, 3}); DataBuffer blank = Nd4j.createBuffer(new double[] {0, 0, 0}); blank.assign(one, twoThree); assertArrayEquals(assertion.asFloat(), blank.asFloat(), 0.0001f); }
Example 6
Source File: FloatDataBufferTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testReadWrite() throws Exception { DataBuffer assertion = Nd4j.createBuffer(new double[] {1, 2, 3}); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); assertion.write(dos); DataBuffer clone = assertion.dup(); val stream = new DataInputStream(new ByteArrayInputStream(bos.toByteArray())); val header = BaseDataBuffer.readHeader(stream); assertion.read(stream, header.getLeft(), header.getMiddle(), header.getRight()); assertArrayEquals(assertion.asFloat(), clone.asFloat(), 0.0001f); }
Example 7
Source File: BaseNDArray.java From nd4j with Apache License 2.0 | 5 votes |
protected static DataBuffer internalCreateBuffer(float[] data) { val perfX = PerformanceTracker.getInstance().helperStartTransaction(); val buffer = Nd4j.createBuffer(data); PerformanceTracker.getInstance().helperRegisterTransaction(0, perfX, data.length * Nd4j.sizeOfDataType(), MemcpyDirection.HOST_TO_HOST); return buffer; }
Example 8
Source File: DoubleDataBufferTest.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testReallocation() { DataBuffer buffer = Nd4j.createBuffer(new double[] {1, 2, 3, 4}); assertEquals(4, buffer.capacity()); double[] old = buffer.asDouble(); buffer.reallocate(6); assertEquals(6, buffer.capacity()); assertArrayEquals(old, buffer.asDouble(), 1e-1); }
Example 9
Source File: FloatDataBufferTest.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testGetSet() throws Exception { float[] d1 = new float[] {1, 2, 3, 4}; DataBuffer d = Nd4j.createBuffer(d1); float[] d2 = d.asFloat(); assertArrayEquals(getFailureMessage(), d1, d2, 1e-1f); }
Example 10
Source File: TestNDArrayCreation.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testBufferCreation() { DataBuffer dataBuffer = Nd4j.createBuffer(new double[] {1, 2}); Pointer pointer = dataBuffer.pointer(); FloatPointer floatPointer = new FloatPointer(pointer); DataBuffer dataBuffer1 = Nd4j.createBuffer(floatPointer, 2); assertEquals(2, dataBuffer1.length()); assertEquals(1.0, dataBuffer1.getDouble(0), 1e-1); assertEquals(2.0, dataBuffer1.getDouble(1), 1e-1); INDArray arr = Nd4j.create(dataBuffer1); System.out.println(arr); }
Example 11
Source File: FloatDataBufferTest.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testPut() throws Exception { float[] d1 = new float[] {1, 2, 3, 4}; DataBuffer d = Nd4j.createBuffer(d1); d.put(0, 0.0); float[] result = new float[] {0, 2, 3, 4}; d1 = d.asFloat(); assertArrayEquals(getFailureMessage(), d1, result, 1e-1f); }
Example 12
Source File: CpuNDArrayFactory.java From nd4j with Apache License 2.0 | 4 votes |
@Override public INDArray create(float[] data, long[] shape, long[] stride, long offset, char ordering) { return new NDArray(Nd4j.createBuffer(data), shape, stride, offset, ordering); }
Example 13
Source File: BaseNDArray.java From deeplearning4j with Apache License 2.0 | 4 votes |
public BaseNDArray(long[] shape, long[] stride, long offset, char ordering) { this(Nd4j.createBuffer(shape.length == 0 ? 1 : ArrayUtil.prodLong(shape)), shape, stride, offset, ordering); }
Example 14
Source File: IntDataBufferTests.java From nd4j with Apache License 2.0 | 4 votes |
@Test public void testBasicSerde1() throws Exception { DataBuffer dataBuffer = Nd4j.createBuffer(new int[] {1, 2, 3, 4, 5}); DataBuffer shapeBuffer = Nd4j.getShapeInfoProvider().createShapeInformation(new int[] {1, 5}).getFirst(); INDArray intArray = Nd4j.createArrayFromShapeBuffer(dataBuffer, shapeBuffer); File tempFile = File.createTempFile("test", "test"); tempFile.deleteOnExit(); Nd4j.saveBinary(intArray, tempFile); InputStream stream = new FileInputStream(tempFile); BufferedInputStream bis = new BufferedInputStream(stream); DataInputStream dis = new DataInputStream(bis); INDArray loaded = Nd4j.read(dis); assertEquals(DataBuffer.Type.INT, loaded.data().dataType()); assertEquals(DataBuffer.Type.LONG, loaded.shapeInfoDataBuffer().dataType()); assertEquals(intArray.data().length(), loaded.data().length()); assertArrayEquals(intArray.data().asInt(), loaded.data().asInt()); }
Example 15
Source File: CudaThreshold.java From nd4j with Apache License 2.0 | 4 votes |
@Override public DataBuffer decompress(DataBuffer buffer) { if (buffer.dataType() != DataBuffer.Type.INT) throw new UnsupportedOperationException(); long compressedLength = buffer.getInt(0); long originalLength = buffer.getInt(1); DataBuffer result = Nd4j.createBuffer(originalLength); CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext(); PointerPointer extras = new PointerPointer(32).put(1, context.getOldStream()); //log.info("DEC Source length: {}", buffer.length()); //log.info("DEC Source: {}", Arrays.toString(buffer.asInt())); NativeOpsHolder.getInstance().getDeviceNativeOps().decodeThresholdFloat(extras, AtomicAllocator.getInstance().getPointer(buffer), compressedLength, (FloatPointer) AtomicAllocator.getInstance().getPointer(result)); AtomicAllocator.getInstance().getAllocationPoint(result).tickDeviceWrite(); //DataBuffer result = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.THRESHOLD, buffer, getGlobalTypeEx()); return result; }
Example 16
Source File: CudaHalfDataBufferTest.java From nd4j with Apache License 2.0 | 3 votes |
@Test public void testSerialization2() throws Exception { DataBuffer bufferOriginal = new CudaFloatDataBuffer(new float[]{1f, 2f, 3f, 4f, 5f}); DataBuffer bufferHalfs = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.FLOAT, bufferOriginal, DataBuffer.TypeEx.FLOAT16); DataTypeUtil.setDTypeForContext(DataBuffer.Type.HALF); File tempFile = File.createTempFile("alpha", "11"); tempFile.deleteOnExit(); // now we serialize halfs, and we expect it to become floats on other side try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(tempFile.getAbsolutePath())))){ bufferHalfs.write(dos); } // loading data back from file DataInputStream dis = new DataInputStream(new FileInputStream(tempFile.getAbsoluteFile())); DataBuffer bufferRestored = Nd4j.createBuffer(bufferOriginal.length()); bufferRestored.read(dis); assertEquals(bufferRestored.dataType(), DataBuffer.Type.HALF); DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT); DataBuffer bufferConverted = Nd4j.getNDArrayFactory().convertDataEx(DataBuffer.TypeEx.FLOAT16, bufferRestored, DataBuffer.TypeEx.FLOAT); assertArrayEquals(bufferOriginal.asFloat(), bufferConverted.asFloat(), 0.01f); }
Example 17
Source File: Shape.java From nd4j with Apache License 2.0 | 2 votes |
/** * Get the shape from * the given int buffer * @param buffer the buffer to get the shape information for * @return */ public static DataBuffer shapeOf(DataBuffer buffer) { int rank = (int) buffer.getLong(0); return Nd4j.createBuffer(buffer, 1, rank); }
Example 18
Source File: BaseNDArray.java From nd4j with Apache License 2.0 | 2 votes |
/** * * @param buffer * @param shape * @param offset */ public BaseNDArray(DataBuffer buffer, int[] shape, long offset) { this(Nd4j.createBuffer(buffer, offset, ArrayUtil.prodLong(shape)), shape, Nd4j.getStrides(shape), offset, Nd4j.order()); }
Example 19
Source File: BaseNDArray.java From deeplearning4j with Apache License 2.0 | 2 votes |
/** * Construct an ndarray of the specified shape * with an empty data array * * @param shape the shape of the ndarray * @param stride the stride of the ndarray * @param offset the desired offset * @param ordering the ordering of the ndarray */ public BaseNDArray(int[] shape, int[] stride, long offset, char ordering) { this(Nd4j.createBuffer(shape.length == 0 ? 1 : ArrayUtil.prodLong(shape)), shape, stride, offset, ordering); }
Example 20
Source File: BaseComplexNDArray.java From nd4j with Apache License 2.0 | 2 votes |
/** * * @param shape * @param offset * @param ordering */ public BaseComplexNDArray(int[] shape, long offset, char ordering) { this(Nd4j.createBuffer(ArrayUtil.prodLong(shape) * 2), shape, Nd4j.getComplexStrides(shape, ordering), offset, ordering); }