Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#isCompressed()

The following examples show how to use org.nd4j.linalg.api.ndarray.INDArray#isCompressed() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SmartFancyBlockingQueue.java From deeplearning4j with Apache License 2.0

6 votes

protected INDArray smartDecompress(INDArray encoded, INDArray target) {
    INDArray result = target == null ? Nd4j.create(paramsShape, paramsOrder) : target;

    if (encoded.isCompressed() || encoded.data().dataType() == DataType.INT) {
        int encoding = encoded.data().getInt(3);
        if (encoding == ThresholdCompression.FLEXIBLE_ENCODING) {
            Nd4j.getExecutioner().thresholdDecode(encoded, result);
        } else if (encoding == ThresholdCompression.BITMAP_ENCODING) {
            Nd4j.getExecutioner().bitmapDecode(encoded, result);
        } else
            throw new ND4JIllegalStateException("Unknown encoding mode: [" + encoding + "]");
    } else {
        result.addi(encoded);
    }

    return result;
}

Example 2

Source File: IndexedTail.java From deeplearning4j with Apache License 2.0

6 votes

protected INDArray smartDecompress(INDArray encoded, @NonNull INDArray target) {
    INDArray result = target;

    if (encoded.isCompressed() || encoded.data().dataType() == DataType.INT) {
        int encoding = encoded.data().getInt(3);
        if (encoding == ThresholdCompression.FLEXIBLE_ENCODING) {
            Nd4j.getExecutioner().thresholdDecode(encoded, result);
        } else if (encoding == ThresholdCompression.BITMAP_ENCODING) {
            Nd4j.getExecutioner().bitmapDecode(encoded, result);
        } else
            throw new ND4JIllegalStateException("Unknown encoding mode: [" + encoding + "]");
    } else {
        result.addi(encoded);
    }

    return result;
}

Example 3

Source File: AbstractCompressor.java From nd4j with Apache License 2.0

5 votes

@Override
public void decompressi(INDArray array) {
    if (!array.isCompressed())
        return;

    array.markAsCompressed(false);
    array.setData(decompress(array.data()));
}

Example 4

Source File: CpuNDArrayFactory.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public INDArray[] tear(INDArray tensor, int... dimensions) {
    if (tensor.isCompressed())
        Nd4j.getCompressor().decompressi(tensor);

    Arrays.sort(dimensions);

    Pair<DataBuffer, DataBuffer> tadBuffers = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(tensor, dimensions);

    long tadLength = 1;
    long[] shape = new long[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        tadLength *= tensor.shape()[dimensions[i]];
        shape[i] = tensor.shape()[dimensions[i]];
    }



    int numTads = (int)(tensor.length() / tadLength);
    INDArray[] result = new INDArray[numTads];

    PointerPointer targets = new PointerPointer(numTads);

    for (int x = 0; x < numTads; x++) {
        result[x] = Nd4j.createUninitialized(shape);

        targets.put(x, result[x].data().pointer());
    }

        nativeOps.tear(null,
                ((BaseCpuDataBuffer) tensor.data()).getOpaqueDataBuffer(), (LongPointer) tensor.shapeInfoDataBuffer().pointer(), null,
                targets, (LongPointer) result[0].shapeInfoDataBuffer().pointer(),
                (LongPointer) tadBuffers.getFirst().pointer(), new LongPointerWrapper(tadBuffers.getSecond().pointer())
        );

    if (nativeOps.lastErrorCode() != 0)
        throw new RuntimeException(nativeOps.lastErrorMessage());

    return result;
}

Example 5

Source File: AbstractCompressor.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public void decompressi(INDArray array) {
    if (!array.isCompressed())
        return;

    array.markAsCompressed(false);
    array.setData(decompress(array.data(), ((CompressedDataBuffer)array.data()).getCompressionDescriptor().getOriginalDataType()));
}

Example 6

Source File: AbstractCompressor.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public INDArray decompress(INDArray array) {
    if (!array.isCompressed())
        return array;

    val descriptor = ((CompressedDataBuffer)array.data()).getCompressionDescriptor();
    val buffer = decompress(array.data(), descriptor.getOriginalDataType());
    val shapeInfo = array.shapeInfoDataBuffer();
    INDArray rest = Nd4j.createArrayFromShapeBuffer(buffer, shapeInfo);

    return rest;
}

Example 7

Source File: BaseScalarOp.java From deeplearning4j with Apache License 2.0

5 votes

public BaseScalarOp(INDArray x, INDArray y, INDArray z, Number num) {
    super(x, y, z);
    if (x.isCompressed())
        Nd4j.getCompressor().decompressi(x);

    try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
        this.scalarValue = Nd4j.scalar(x.dataType(), num);
    }
}

Example 8

Source File: BaseScalarOp.java From deeplearning4j with Apache License 2.0

5 votes

public BaseScalarOp(INDArray x, Number num) {
    super(x);
    if (x.isCompressed())
        Nd4j.getCompressor().decompressi(x);

    try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
        this.scalarValue = Nd4j.scalar(x.dataType(), num);
    }

}

Example 9

Source File: BaseScalarOp.java From deeplearning4j with Apache License 2.0

5 votes

public BaseScalarOp(INDArray x, INDArray z, Number set) {
    super(x, null, z);
    if (x.isCompressed())
        Nd4j.getCompressor().decompressi(x);

    try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
        this.scalarValue = Nd4j.scalar(x.dataType(), set);
    }
}

Example 10

Source File: JCublasNDArrayFactory.java From nd4j with Apache License 2.0

4 votes

public INDArray[] tear(INDArray tensor, int... dimensions) {
    if (tensor.isCompressed())
        Nd4j.getCompressor().decompressi(tensor);

    Arrays.sort(dimensions);

    Pair<DataBuffer, DataBuffer> tadBuffers = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(tensor, dimensions);

    long tadLength = 1;
    val shape = new long[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        tadLength *= tensor.shape()[dimensions[i]];
        shape[i] = tensor.shape()[dimensions[i]];
    }


    int numTads = (int)(tensor.lengthLong() / tadLength);
    INDArray[] result = new INDArray[numTads];

    long[] xPointers = new long[numTads];

    CudaContext context = AtomicAllocator.getInstance().getFlowController().prepareAction(null, tensor);

    for (int x = 0; x < numTads; x++) {
        result[x] = Nd4j.createUninitialized(shape);

        context = AtomicAllocator.getInstance().getFlowController().prepareAction(result[x]);

        xPointers[x] = AtomicAllocator.getInstance().getPointer(result[x], context).address();
    }

    CudaDoubleDataBuffer tempX = new CudaDoubleDataBuffer(numTads);

    AtomicAllocator.getInstance().memcpyBlocking(tempX, new LongPointer(xPointers), xPointers.length * 8, 0);

    PointerPointer extraz = new PointerPointer(null, // not used
            context.getOldStream(), AtomicAllocator.getInstance().getDeviceIdPointer());

    if (Nd4j.dataType() == DataBuffer.Type.DOUBLE) {
        nativeOps.tearDouble(extraz,
                (DoublePointer) AtomicAllocator.getInstance().getPointer(tensor, context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tensor.shapeInfoDataBuffer(), context),
                new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)),
                (LongPointer) AtomicAllocator.getInstance().getPointer(result[0].shapeInfoDataBuffer(), context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context),
                new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context))
        );
    } else if (Nd4j.dataType() == DataBuffer.Type.FLOAT) {
        nativeOps.tearFloat(extraz,
                (FloatPointer) AtomicAllocator.getInstance().getPointer(tensor, context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tensor.shapeInfoDataBuffer(), context),
                new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)),
                (LongPointer) AtomicAllocator.getInstance().getPointer(result[0].shapeInfoDataBuffer(), context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context),
                new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context))
        );
    } else if (Nd4j.dataType() == DataBuffer.Type.HALF) {
        nativeOps.tearHalf(extraz,
                (ShortPointer) AtomicAllocator.getInstance().getPointer(tensor, context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tensor.shapeInfoDataBuffer(), context),
                new PointerPointer(AtomicAllocator.getInstance().getPointer(tempX, context)),
                (LongPointer) AtomicAllocator.getInstance().getPointer(result[0].shapeInfoDataBuffer(), context),
                (LongPointer) AtomicAllocator.getInstance().getPointer(tadBuffers.getFirst(), context),
                new LongPointerWrapper(AtomicAllocator.getInstance().getPointer(tadBuffers.getSecond(), context))
        );
    }

    AtomicAllocator.getInstance().getFlowController().registerActionAllWrite(context, result);
    AtomicAllocator.getInstance().getFlowController().registerAction(context,null, result);

    return result;
}

Example 11

Source File: CpuNDArrayFactory.java From nd4j with Apache License 2.0

4 votes

public INDArray[] tear(INDArray tensor, int... dimensions) {
    if (tensor.isCompressed())
        Nd4j.getCompressor().decompressi(tensor);

    Arrays.sort(dimensions);

    Pair<DataBuffer, DataBuffer> tadBuffers = Nd4j.getExecutioner().getTADManager().getTADOnlyShapeInfo(tensor, dimensions);

    long tadLength = 1;
    long[] shape = new long[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        tadLength *= tensor.shape()[dimensions[i]];
        shape[i] = tensor.shape()[dimensions[i]];
    }



    int numTads = (int)(tensor.lengthLong() / tadLength);
    INDArray[] result = new INDArray[numTads];

    PointerPointer targets = new PointerPointer(numTads);

    for (int x = 0; x < numTads; x++) {
        result[x] = Nd4j.createUninitialized(shape);

        targets.put(x, result[x].data().pointer());
    }

    if (Nd4j.dataType() == DataBuffer.Type.DOUBLE) {
        nativeOps.tearDouble(null,
                (DoublePointer) tensor.data().pointer(),
                (LongPointer) tensor.shapeInfoDataBuffer().pointer(),
                targets,
                (LongPointer) result[0].shapeInfoDataBuffer().pointer(),
                (LongPointer) tadBuffers.getFirst().pointer(),
                new LongPointerWrapper(tadBuffers.getSecond().pointer())
        );
    } else if (Nd4j.dataType() == DataBuffer.Type.FLOAT) {
        nativeOps.tearFloat(null,
                (FloatPointer) tensor.data().pointer(),
                (LongPointer) tensor.shapeInfoDataBuffer().pointer(),
                targets,
                (LongPointer) result[0].shapeInfoDataBuffer().pointer(),
                (LongPointer) tadBuffers.getFirst().pointer(),
                new LongPointerWrapper(tadBuffers.getSecond().pointer())
                );
    } else if (Nd4j.dataType() == DataBuffer.Type.HALF) {
        throw new UnsupportedOperationException("Half precision isn't supported for CPU backend");
    }

    return result;
}

Example 12

Source File: BasicNDArrayCompressor.java From nd4j with Apache License 2.0

4 votes

/**
 *
 * @param array
 */
public void autoDecompress(INDArray array) {
    if (array.isCompressed())
        decompressi(array);
}

Example 13

Source File: BasicNDArrayCompressor.java From deeplearning4j with Apache License 2.0

4 votes

/**
 *
 * @param array
 */
public void autoDecompress(INDArray array) {
    if (array.isCompressed())
        decompressi(array);
}