org.nd4j.serde.binary.BinarySerde Java Examples

@Test
public void testServer() throws Exception {
    int localPort = getAvailablePort();
    Nd4j.getRandom().setSeed(7);
    INDArray rand = Nd4j.randn(10, 5);
    File writeToTmp = testDir.newFile();
    writeToTmp.deleteOnExit();
    BinarySerde.writeArrayToDisk(rand, writeToTmp);
    NearestNeighborsServer.runMain("--ndarrayPath", writeToTmp.getAbsolutePath(), "--nearestNeighborsPort",
            String.valueOf(localPort));

    Thread.sleep(3000);

    NearestNeighborsClient client = new NearestNeighborsClient("http://localhost:" + localPort);
    NearestNeighborsResults result = client.knnNew(5, rand.getRow(0));
    assertEquals(5, result.getResults().size());
    NearestNeighborsServer.getInstance().stop();
}
 

private INDArray getOrSetArrForContext() {
    String path = memMapConfig.getArrayPath();


    File loadFrom = new File(path);
    if (!loadFrom.exists()) {
        throw new IllegalStateException("File not found at path " + path);
    }

    if (mArray.get() == null) {
        try (MemoryWorkspace ignored = Nd4j.getWorkspaceManager().getAndActivateWorkspace(mmap, memMapConfig.getWorkSpaceName())) {
            if (path.endsWith("npy"))
                mArray.set((Nd4j.createFromNpyFile(loadFrom)));
            else {
                mArray.set(BinarySerde.readFromDisk(loadFrom));
            }
        } catch (IOException e) {
            log.error("Error creating nd4j array", e);
        }
    }

    return mArray.get();
}
 

/**
 * Load an ndarray from a blob
 *
 * @param blob the blob to load from
 * @return the loaded ndarray
 */
@Override
public INDArray load(Blob blob) throws SQLException {
    if (blob == null)
        return null;
    try(InputStream is = blob.getBinaryStream()) {
        ByteBuffer direct = ByteBuffer.allocateDirect((int) blob.length());
        ReadableByteChannel readableByteChannel = Channels.newChannel(is);
        readableByteChannel.read(direct);
        Buffer byteBuffer = (Buffer) direct;
        byteBuffer.rewind();
        return BinarySerde.toArray(direct);
    } catch (Exception e) {
       throw new RuntimeException(e);
    }


}
 

private void doSave(INDArray save, String id) throws SQLException, IOException {
    Connection c = dataSource.getConnection();
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    if (save instanceof IComplexNDArray) {
        IComplexNDArray c2 = (IComplexNDArray) save;
        Nd4j.writeComplex(c2, dos);
    } else {
        BinarySerde.writeArrayToOutputStream(save,bos);
    }

    byte[] bytes = bos.toByteArray();

    PreparedStatement preparedStatement = c.prepareStatement(insertStatement());
    preparedStatement.setString(1, id);
    preparedStatement.setBytes(2, bytes);
    preparedStatement.executeUpdate();


}
 

@Test(timeout = 60000)
public void runAdd(TestContext testContext) throws Exception {
    BatchNd4jInputParserVerticle verticleRef = (BatchNd4jInputParserVerticle) verticle;
    NativeImageLoader nativeImageLoader = new NativeImageLoader();
    Image image = nativeImageLoader.asImageMatrix(new ClassPathResource("data/5.png").getFile());
    File tmpFile = temporary.newFile();
    BinarySerde.writeArrayToDisk(image.getImage(), tmpFile);

    given().port(port)
            .multiPart("input1", tmpFile)
            .when().post("/")
            .then().statusCode(200);
    assertNotNull("Inputs were null. This means parsing failed.", verticleRef.getBatch());
    assertTrue(verticleRef.getBatch().length >= 1);
    assertNotNull(verticleRef.getBatch());
}
 

/**
 * Convert a {@link INDArray}
 * given a {@link Output.DataFormat}
 *
 * @param input      the batch ndarray to convert
 * @param dataFormat the output data format
 * @return converted buffer
 */
public static Buffer convertBatchOutput(INDArray input, Output.DataFormat dataFormat) {
    Preconditions.checkNotNull(input, "Input was null!");
    Preconditions.checkNotNull(dataFormat, "Response output type was null!");
    Buffer ret = null;

    switch (dataFormat) {
        case NUMPY:
            ret = Buffer.buffer(Unpooled.wrappedBuffer(ByteBuffer.wrap(Nd4j.toNpyByteArray(input))));
            break;
        case ND4J:
            ByteBuffer byteBuffer2 = BinarySerde.toByteBuffer(input);
            ret = Buffer.buffer(Unpooled.wrappedBuffer(byteBuffer2));
            break;
        case ARROW:
            Tensor tensor = ArrowSerde.toTensor(input);
            ret = Buffer.buffer(Unpooled.wrappedBuffer(tensor.getByteBuffer()));
            break;
        case JSON:
            ret = Buffer.buffer(input.toStringFull());
            break;
    }

    return ret;
}
 

/**
 * Load an ndarray from a blob
 *
 * @param blob the blob to load from
 * @return the loaded ndarray
 */
@Override
public INDArray load(Blob blob) throws SQLException {
    if (blob == null)
        return null;
    try(InputStream is = blob.getBinaryStream()) {
        ByteBuffer direct = ByteBuffer.allocateDirect((int) blob.length());
        ReadableByteChannel readableByteChannel = Channels.newChannel(is);
        readableByteChannel.read(direct);
        Buffer byteBuffer = (Buffer) direct;
        byteBuffer.rewind();
        return BinarySerde.toArray(direct);
    } catch (Exception e) {
       throw new RuntimeException(e);
    }


}
 

/**
 * Convert an ndarray to a blob
 *
 * @param toConvert the ndarray to convert
 * @return the converted ndarray
 */
@Override
public Blob convert(INDArray toConvert) throws SQLException {
    ByteBuffer byteBuffer = BinarySerde.toByteBuffer(toConvert);
    Buffer buffer = (Buffer) byteBuffer;
    buffer.rewind();
    byte[] arr = new byte[byteBuffer.capacity()];
    byteBuffer.get(arr);
    Connection c = dataSource.getConnection();
    Blob b = c.createBlob();
    b.setBytes(1, arr);
    return b;
}
 

private void doSave(INDArray save, String id) throws SQLException, IOException {
    Connection c = dataSource.getConnection();
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    BinarySerde.writeArrayToOutputStream(save,bos);

    byte[] bytes = bos.toByteArray();

    PreparedStatement preparedStatement = c.prepareStatement(insertStatement());
    preparedStatement.setString(1, id);
    preparedStatement.setBytes(2, bytes);
    preparedStatement.executeUpdate();


}
 

/**
 * Convert an ndarray to a blob
 *
 * @param toConvert the ndarray to convert
 * @return the converted ndarray
 */
@Override
public Blob convert(INDArray toConvert) throws SQLException {
    ByteBuffer byteBuffer = BinarySerde.toByteBuffer(toConvert);
    Buffer buffer = (Buffer) byteBuffer;
    buffer.rewind();
    byte[] arr = new byte[byteBuffer.capacity()];
    byteBuffer.get(arr);
    Connection c = dataSource.getConnection();
    Blob b = c.createBlob();
    b.setBytes(1, arr);
    return b;
}
 

/**
 * Based on an input {@link ColumnType}
 * get an entry from a {@link FieldVector}
 *
 * @param item the row of the item to get from the column vector
 * @param from the column vector from
 * @param columnType the column type
 * @return the resulting writable
 */
public static Writable fromEntry(int item,FieldVector from,ColumnType columnType) {
    if(from.getValueCount() < item) {
        throw new IllegalArgumentException("Index specified greater than the number of items in the vector with length " + from.getValueCount());
    }

    switch(columnType) {
        case Integer:
            return new IntWritable(getIntFromFieldVector(item,from));
        case Long:
            return new LongWritable(getLongFromFieldVector(item,from));
        case Float:
            return new FloatWritable(getFloatFromFieldVector(item,from));
        case Double:
            return new DoubleWritable(getDoubleFromFieldVector(item,from));
        case Boolean:
            BitVector bitVector = (BitVector) from;
            return new BooleanWritable(bitVector.get(item) > 0);
        case Categorical:
            VarCharVector varCharVector = (VarCharVector) from;
            return new Text(varCharVector.get(item));
        case String:
            VarCharVector varCharVector2 = (VarCharVector) from;
            return new Text(varCharVector2.get(item));
        case Time:
            //TODO: need to look at closer
            return new LongWritable(getLongFromFieldVector(item,from));
        case NDArray:
            VarBinaryVector valueVector = (VarBinaryVector) from;
            byte[] bytes = valueVector.get(item);
            ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
            direct.put(bytes);
            INDArray fromTensor = BinarySerde.toArray(direct);
            return new NDArrayWritable(fromTensor);
        default:
            throw new IllegalArgumentException("Illegal type " + from.getClass().getName());
    }
}
 

/**
 * Returns a vector representing a tensor view
 * of each ndarray.
 * Each ndarray will be a "row" represented as a tensor object
 * with in the return {@link VarBinaryVector}
 * @param bufferAllocator the buffer allocator to use
 * @param name the name of the column
 * @param data the input arrays
 * @return
 */
public static VarBinaryVector vectorFor(BufferAllocator bufferAllocator,String name,INDArray[] data) {
    VarBinaryVector ret = new VarBinaryVector(name,bufferAllocator);
    ret.allocateNew();
    for(int i = 0; i < data.length; i++) {
        //slice the databuffer to use only the needed portion of the buffer
        //for proper offset
        ByteBuffer byteBuffer = BinarySerde.toByteBuffer(data[i]);
        ret.set(i,byteBuffer,0,byteBuffer.capacity());
    }

    return ret;
}
 

@Test
public void runAdd(TestContext testContext) throws Exception {
    INDArray x = Nd4j.create(new float[]{1.0f, 2.0f});
    INDArray y = Nd4j.create(new float[]{2.0f, 3.0f});
    byte[] xNpy = Nd4j.toNpyByteArray(x);
    byte[] yNpy = Nd4j.toNpyByteArray(y);


    File xFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(xFile, xNpy);

    File yFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(yFile, yNpy);


    Response response = given().port(port)
            .multiPart("x", xFile)
            .multiPart("y", yFile)
            .post("/nd4j/numpy")
            .andReturn();

    assertEquals("Response failed", 200, response.getStatusCode());

    INDArray bodyResult = BinarySerde.toArray(ByteBuffer.wrap(response.getBody().asByteArray()));
    assertArrayEquals(new long[]{2}, bodyResult.shape());
    assertEquals(Nd4j.create(new float[]{3.0f, 5.0f}), bodyResult);


}
 

@Test
public void runAdd(TestContext testContext) throws Exception {
    INDArray x = Nd4j.create(new float[]{1.0f, 2.0f});
    INDArray y = Nd4j.create(new float[]{2.0f, 3.0f});
    byte[] xNpy = Nd4j.toNpyByteArray(x);
    byte[] yNpy = Nd4j.toNpyByteArray(y);


    File xFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(xFile, xNpy);

    File yFile = temporary.newFile();
    FileUtils.writeByteArrayToFile(yFile, yNpy);


    Response response = given().port(port)
            .multiPart("x", xFile)
            .multiPart("y", yFile)
            .post("/nd4j/numpy")
            .andReturn();

    assertEquals("Response failed", 200, response.getStatusCode());

    INDArray bodyResult = BinarySerde.toArray(ByteBuffer.wrap(response.getBody().asByteArray()));
    assertArrayEquals(new long[]{2}, bodyResult.shape());
    assertEquals(Nd4j.create(new float[]{3.0f, 5.0f}), bodyResult);

    given().port(port).get("/metrics")
            .then()
            .statusCode(200).and()
            .contentType("text/plain");

    Response response1 = given().port(port).get("/metrics")
            .andReturn();

    System.out.println(response1.asString());

}
 

public static Writable fromEntry(int item, FieldVector from, ColumnType columnType) {
    if (from.getValueCount() < item) {
        throw new IllegalArgumentException("Index specified greater than the number of items in the vector with length " + from.getValueCount());
    } else {
        switch (columnType) {
            case Integer:
                return new IntWritable(getIntFromFieldVector(item, from));
            case Float:
                return new FloatWritable(getFloatFromFieldVector(item, from));
            case Double:
                return new DoubleWritable(getDoubleFromFieldVector(item, from));
            case Long:
                return new LongWritable(getLongFromFieldVector(item, from));
            case NDArray:
                VarBinaryVector valueVector = (VarBinaryVector) from;
                byte[] bytes = valueVector.get(item);
                ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
                direct.put(bytes);
                INDArray fromTensor = BinarySerde.toArray(direct);
                return new NDArrayWritable(fromTensor);
            case Boolean:
                BitVector bitVector = (BitVector) from;
                return new BooleanWritable(bitVector.get(item) > 0);
            case Categorical:
                VarCharVector varCharVector = (VarCharVector) from;
                return new Text(varCharVector.get(item));
            case Time:
                return new LongWritable(getLongFromFieldVector(item, from));
            case Bytes:
            default:
                throw new IllegalArgumentException("Illegal type " + from.getClass().getName());
            case String:
                VarCharVector varCharVector2 = (VarCharVector) from;
                return new Text(varCharVector2.get(item));
        }
    }
}
 

public static VarBinaryVector vectorFor(BufferAllocator bufferAllocator, String name, INDArray[] data) {
    VarBinaryVector ret = new VarBinaryVector(name, bufferAllocator);
    ret.allocateNew();

    for (int i = 0; i < data.length; ++i) {
        ByteBuffer byteBuffer = BinarySerde.toByteBuffer(data[i]);
        ret.set(i, byteBuffer, 0, byteBuffer.capacity());
    }

    return ret;
}
 

/**
 * Convert a {@link Buffer}
 * to an {@link INDArray}
 * using one of three types:
 * numpy: (converts using {@link Nd4j#createNpyFromByteArray(byte[])}
 * nd4j: (converts using {@link BinarySerde#toArray(ByteBuffer)}
 * with a direct byte buffer copy (nd4j requires direct allocation
 * for byte buffers
 * json:  (converts with a straight for loop, note that this only supports matrices only)
 *
 * @param buffer the buffer to convert
 * @param type   the type of buffer
 * @return the created ndarray
 */
public static INDArray toArray(Buffer buffer, String type) {
    INDArray trueFeedback = null;
    switch (type) {
        case "numpy":
            trueFeedback = Nd4j.createNpyFromByteArray(buffer.getBytes());
            break;
        case "nd4j":
            ByteBuffer direct = ByteBuffer.allocateDirect(buffer.length());
            direct.put(buffer.getBytes());
            direct.rewind();
            trueFeedback = BinarySerde.toArray(direct);
            break;
        case "json":
            JsonArray jsonArray = new JsonArray(buffer.toString());
            INDArray arr = Nd4j.create(jsonArray.size(), jsonArray.getJsonArray(0).size());
            for (int i = 0; i < arr.rows(); i++) {
                for (int j = 0; j < arr.columns(); j++) {
                    arr.putScalar(i, j, jsonArray.getJsonArray(i).getDouble(j));
                }
            }

            trueFeedback = arr;
            break;
        default:
            throw new IllegalArgumentException("Illegal type " + type);

    }

    return trueFeedback;
}
 

@Override
public NDArrayWritable convert(Buffer input, ConverterArgs parameters, Map<String, Object> contextData) {
    Preconditions.checkState(input.length() > 0, "Buffer appears to be empty!");
    INDArray fromNpyPointer = BinarySerde.toArray(input.getByteBuf().nioBuffer());

    //permute required
    if (parameters != null && parameters.getImageProcessingInitialLayout() != null && !parameters.getImageProcessingInitialLayout().equals(parameters.getImageProcessingRequiredLayout())) {
        fromNpyPointer = ImagePermuter.permuteOrder(fromNpyPointer, parameters.getImageProcessingInitialLayout(), parameters.getImageProcessingRequiredLayout());
    }

    return new NDArrayWritable(fromNpyPointer);
}
 

public static void setValue(ColumnType columnType, FieldVector fieldVector, Object value, int row) {
    if (!(value instanceof NullWritable)) {
        try {
            switch (columnType) {
                case Integer:
                    int set;
                    if (fieldVector instanceof IntVector) {
                        IntVector intVector = (IntVector) fieldVector;
                        set = TypeConversion.getInstance().convertInt(value);
                        intVector.set(row, set);
                    } else {
                        if (!(fieldVector instanceof UInt4Vector)) {
                            throw new UnsupportedOperationException("Illegal type " + fieldVector.getClass() + " for int type");
                        }

                        UInt4Vector uInt4Vector = (UInt4Vector) fieldVector;
                        set = TypeConversion.getInstance().convertInt(value);
                        uInt4Vector.set(row, set);
                    }
                    break;
                case Float:
                    Float4Vector float4Vector = (Float4Vector) fieldVector;
                    float set2 = TypeConversion.getInstance().convertFloat(value);
                    float4Vector.set(row, set2);
                    break;
                case Double:
                    double set3 = TypeConversion.getInstance().convertDouble(value);
                    Float8Vector float8Vector = (Float8Vector) fieldVector;
                    float8Vector.set(row, set3);
                    break;
                case Long:
                    if (fieldVector instanceof BigIntVector) {
                        BigIntVector largeIntVector = (BigIntVector) fieldVector;
                        largeIntVector.set(row, TypeConversion.getInstance().convertLong(value));
                    } else {
                        if (!(fieldVector instanceof UInt8Vector)) {
                            throw new UnsupportedOperationException("Illegal type " + fieldVector.getClass() + " for long type");
                        }

                        UInt8Vector uInt8Vector = (UInt8Vector) fieldVector;
                        uInt8Vector.set(row, TypeConversion.getInstance().convertLong(value));
                    }
                    break;
                case NDArray:
                    NDArrayWritable arr = (NDArrayWritable) value;
                    VarBinaryVector nd4jArrayVector = (VarBinaryVector) fieldVector;
                    ByteBuffer byteBuffer = BinarySerde.toByteBuffer(arr.get());
                    nd4jArrayVector.setSafe(row, byteBuffer, 0, byteBuffer.capacity());
                case Boolean:
                case Bytes:
                default:
                    break;
                case Categorical:
                case String:
                    String stringSet = TypeConversion.getInstance().convertString(value);
                    VarCharVector textVector = (VarCharVector) fieldVector;
                    textVector.setSafe(row, stringSet.getBytes());
                    break;
                case Time:
                    long timeSet = TypeConversion.getInstance().convertLong(value);
                    setLongInTime(fieldVector, row, timeSet);
            }
        } catch (Exception var16) {
            log.warn("Unable to set value at row " + row);
        }

    }
}
 

@Override
public void start() throws Exception {
    instance = this;

    String[] pathArr = instanceArgs.ndarrayPath.split(",");
    //INDArray[] pointsArr = new INDArray[pathArr.length];
    // first of all we reading shapes of saved eariler files
    int rows = 0;
    int cols = 0;
    for (int i = 0; i < pathArr.length; i++) {
        DataBuffer shape = BinarySerde.readShapeFromDisk(new File(pathArr[i]));

        log.info("Loading shape {} of {}; Shape: [{} x {}]", i + 1, pathArr.length, Shape.size(shape, 0),
                Shape.size(shape, 1));

        if (Shape.rank(shape) != 2)
            throw new DL4JInvalidInputException("NearestNeighborsServer assumes 2D chunks");

        rows += Shape.size(shape, 0);

        if (cols == 0)
            cols = Shape.size(shape, 1);
        else if (cols != Shape.size(shape, 1))
            throw new DL4JInvalidInputException(
                    "NearestNeighborsServer requires equal 2D chunks. Got columns mismatch.");
    }

    final List<String> labels = new ArrayList<>();
    if (instanceArgs.labelsPath != null) {
        String[] labelsPathArr = instanceArgs.labelsPath.split(",");
        for (int i = 0; i < labelsPathArr.length; i++) {
            labels.addAll(FileUtils.readLines(new File(labelsPathArr[i]), "utf-8"));
        }
    }
    if (!labels.isEmpty() && labels.size() != rows)
        throw new DL4JInvalidInputException(String.format("Number of labels must match number of rows in points matrix (expected %d, found %d)", rows, labels.size()));

    final INDArray points = Nd4j.createUninitialized(rows, cols);

    int lastPosition = 0;
    for (int i = 0; i < pathArr.length; i++) {
        log.info("Loading chunk {} of {}", i + 1, pathArr.length);
        INDArray pointsArr = BinarySerde.readFromDisk(new File(pathArr[i]));

        points.get(NDArrayIndex.interval(lastPosition, lastPosition + pointsArr.rows())).assign(pointsArr);
        lastPosition += pointsArr.rows();

        // let's ensure we don't bring too much stuff in next loop
        System.gc();
    }

    VPTree tree = new VPTree(points, instanceArgs.similarityFunction, instanceArgs.invert);

    //Set play secret key, if required
    //http://www.playframework.com/documentation/latest/ApplicationSecret
    String crypto = System.getProperty("play.crypto.secret");
    if (crypto == null || "changeme".equals(crypto) || "".equals(crypto)) {
        byte[] newCrypto = new byte[1024];

        new Random().nextBytes(newCrypto);

        String base64 = Base64.getEncoder().encodeToString(newCrypto);
        System.setProperty("play.crypto.secret", base64);
    }

    Router r = Router.router(vertx);
    r.route().handler(BodyHandler.create());  //NOTE: Setting this is required to receive request body content at all
    createRoutes(r, labels, tree, points);

    vertx.createHttpServer()
            .requestHandler(r)
            .listen(instanceArgs.port);
}