Java Code Examples for org.nd4j.linalg.factory.Nd4j#write()

@Override
public void write(@NonNull NormalizerMinMaxScaler normalizer, @NonNull OutputStream stream) throws IOException {
    try (DataOutputStream dos = new DataOutputStream(stream)) {
        dos.writeBoolean(normalizer.isFitLabel());
        dos.writeDouble(normalizer.getTargetMin());
        dos.writeDouble(normalizer.getTargetMax());

        Nd4j.write(normalizer.getMin(), dos);
        Nd4j.write(normalizer.getMax(), dos);

        if (normalizer.isFitLabel()) {
            Nd4j.write(normalizer.getLabelMin(), dos);
            Nd4j.write(normalizer.getLabelMax(), dos);
        }
        dos.flush();
    }
}
 

@Test
public void testSerialization() throws Exception {
    Nd4j.getRandom().setSeed(12345);
    INDArray arr = Nd4j.rand(1,20);

    String temp = System.getProperty("java.io.tmpdir");

    String outPath = FilenameUtils.concat(temp,"dl4jtestserialization.bin");

    try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(outPath)))){
        Nd4j.write(arr,dos);
    }

    INDArray in;
    try(DataInputStream dis = new DataInputStream(new FileInputStream(outPath))){
        in = Nd4j.read(dis);
    }

    INDArray inDup = in.dup();

    System.out.println(in);
    System.out.println(inDup);

    assertEquals(arr,in);       //Passes:   Original array "in" is OK, but array "inDup" is not!?
    assertEquals(in,inDup);     //Fails
}
 

private String[] writeData(DataSet write) throws IOException {
    String[] ret = new String[2];
    String dataSetId = UUID.randomUUID().toString();
    BufferedOutputStream dataOut =
                    new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".bin")));
    DataOutputStream dos = new DataOutputStream(dataOut);
    Nd4j.write(write.getFeatures(), dos);
    dos.flush();
    dos.close();


    BufferedOutputStream dataOutLabels =
                    new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".labels.bin")));
    DataOutputStream dosLabels = new DataOutputStream(dataOutLabels);
    Nd4j.write(write.getLabels(), dosLabels);
    dosLabels.flush();
    dosLabels.close();
    ret[0] = new File(rootDir, dataSetId + ".bin").getAbsolutePath();
    ret[1] = new File(rootDir, dataSetId + ".labels.bin").getAbsolutePath();
    return ret;

}
 

/**
 * Serialize a MultiNormalizerStandardize to a output stream
 *
 * @param normalizer the normalizer
 * @param stream     the output stream to write to
 * @throws IOException
 */
public void write(@NonNull MultiNormalizerStandardize normalizer, @NonNull OutputStream stream) throws IOException {
    try (DataOutputStream dos = new DataOutputStream(stream)) {
        dos.writeBoolean(normalizer.isFitLabel());
        dos.writeInt(normalizer.numInputs());
        dos.writeInt(normalizer.isFitLabel() ? normalizer.numOutputs() : -1);

        for (int i = 0; i < normalizer.numInputs(); i++) {
            Nd4j.write(normalizer.getFeatureMean(i), dos);
            Nd4j.write(normalizer.getFeatureStd(i), dos);
        }
        if (normalizer.isFitLabel()) {
            for (int i = 0; i < normalizer.numOutputs(); i++) {
                Nd4j.write(normalizer.getLabelMean(i), dos);
                Nd4j.write(normalizer.getLabelStd(i), dos);
            }
        }
        dos.flush();
    }
}
 

@Test
public void timeOldVsNew() throws Exception {
    int numTrials = 1000;
    long oldTotal = 0;
    long newTotal = 0;
    INDArray arr = Nd4j.create(100000);
    Nd4j.getCompressor().compressi(arr, "GZIP");
    for (int i = 0; i < numTrials; i++) {
        StopWatch oldStopWatch = new StopWatch();
        // FIXME: int cast
        BufferedOutputStream bos = new BufferedOutputStream(new ByteArrayOutputStream((int) arr.length()));
        DataOutputStream dos = new DataOutputStream(bos);
        oldStopWatch.start();
        Nd4j.write(arr, dos);
        oldStopWatch.stop();
        // System.out.println("Old " + oldStopWatch.getNanoTime());
        oldTotal += oldStopWatch.getNanoTime();
        StopWatch newStopWatch = new StopWatch();
        newStopWatch.start();
        BinarySerde.toByteBuffer(arr);
        newStopWatch.stop();
        //  System.out.println("New " + newStopWatch.getNanoTime());
        newTotal += newStopWatch.getNanoTime();

    }

    oldTotal /= numTrials;
    newTotal /= numTrials;
    System.out.println("Old avg " + oldTotal + " New avg " + newTotal);

}
 

@Test
public void testSerializationOnViewsNd4jWriteRead() throws Exception {
    int length = 100;
    Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.FLOAT);
    INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(sub, dos);
    }
    byte[] bytes = baos.toByteArray();

    //SET DATA TYPE TO DOUBLE and initialize another array with the same contents
    //Nd4j.create(1);
    DataTypeUtil.setDTypeForContext(DataType.DOUBLE);
    System.out.println("The data opType is " + Nd4j.dataType());
    INDArray arr1 = Nd4j.linspace(1, length, length, DataType.FLOAT).reshape('c', 10, 10);
    INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10));

    INDArray arr2;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) {
        arr2 = Nd4j.read(dis);
    }

    //assertEquals(sub,arr2);\
    assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01);
}
 

public static void write(INDArray arr, File f) {
    try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(f)))) {
        Nd4j.write(arr, dos);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testReadWriteDouble() throws Exception {
    INDArray write = Nd4j.linspace(1, 4, 4);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    Nd4j.write(write, dos);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream dis = new DataInputStream(bis);
    INDArray read = Nd4j.read(dis);
    assertEquals(write, read);

}
 

@Test
public void testReadWriteDouble() throws Exception {
    INDArray write = Nd4j.linspace(1, 4, 4, DataType.FLOAT);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    Nd4j.write(write, dos);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    DataInputStream dis = new DataInputStream(bis);
    INDArray read = Nd4j.read(dis);
    assertEquals(write, read);

}
 

@Test
public void timeOldVsNew() throws Exception {
    int numTrials = 1000;
    long oldTotal = 0;
    long newTotal = 0;
    INDArray arr = Nd4j.create(100000);
    Nd4j.getCompressor().compressi(arr, "GZIP");
    for (int i = 0; i < numTrials; i++) {
        StopWatch oldStopWatch = new StopWatch();
        BufferedOutputStream bos = new BufferedOutputStream(new ByteArrayOutputStream((int) arr.length()));
        DataOutputStream dos = new DataOutputStream(bos);
        oldStopWatch.start();
        Nd4j.write(arr, dos);
        oldStopWatch.stop();
        // System.out.println("Old " + oldStopWatch.getNanoTime());
        oldTotal += oldStopWatch.getNanoTime();
        StopWatch newStopWatch = new StopWatch();
        newStopWatch.start();
        AeronNDArraySerde.toBuffer(arr);
        newStopWatch.stop();
        //  System.out.println("New " + newStopWatch.getNanoTime());
        newTotal += newStopWatch.getNanoTime();

    }

    oldTotal /= numTrials;
    newTotal /= numTrials;
    System.out.println("Old avg " + oldTotal + " New avg " + newTotal);

}
 

@Test
public void testSerializationFullArrayNd4jWriteRead() throws Exception {
    int length = 100;
    INDArray arrC = Nd4j.linspace(1, length, length).reshape('c', 10, 10);
    INDArray arrF = Nd4j.linspace(1, length, length).reshape('f', 10, 10);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(arrC, dos);
    }
    byte[] bytesC = baos.toByteArray();
    baos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(baos)) {
        Nd4j.write(arrF, dos);
    }
    byte[] bytesF = baos.toByteArray();

    INDArray arr2C;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesC))) {
        arr2C = Nd4j.read(dis);
    }
    INDArray arr2F;
    try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesF))) {
        arr2F = Nd4j.read(dis);
    }

    assertEquals(arrC, arr2C);
    assertEquals(arrF, arr2F);
}
 

private void saveINDArrays(INDArray[] arrays, DataOutputStream dos, boolean isMask) throws IOException {
    if (arrays != null && arrays.length > 0) {
        for (INDArray fm : arrays) {
            if (isMask && fm == null) {
                INDArray temp = EMPTY_MASK_ARRAY_PLACEHOLDER.get();
                if(temp == null){
                    EMPTY_MASK_ARRAY_PLACEHOLDER.set(Nd4j.create(new float[] {-1}));
                    temp = EMPTY_MASK_ARRAY_PLACEHOLDER.get();
                }
                fm = temp;
            }
            Nd4j.write(fm, dos);
        }
    }
}
 

@Test
public void testWorkspacesSerde2() throws Exception {
    INDArray array = Nd4j.create(10).assign(1.0);
    INDArray restored = null;

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    Nd4j.write(array, dos);

    try (Nd4jWorkspace workspace = (Nd4jWorkspace) Nd4j.getWorkspaceManager()
                    .getAndActivateWorkspace(basicConfiguration, "WS_1")) {
        workspace.enableDebug(true);

        ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
        DataInputStream dis = new DataInputStream(bis);
        restored = Nd4j.read(dis);

        long requiredMemory = 10 * Nd4j.sizeOfDataType();
        assertEquals(requiredMemory + requiredMemory % 8, workspace.getPrimaryOffset());

        assertEquals(array.length(), restored.length());
        assertEquals(1.0f, restored.meanNumber().floatValue(), 1.0f);

        // we want to ensure it's the same cached shapeInfo used here
        assertEquals(array.shapeInfoDataBuffer().addressPointer().address(), restored.shapeInfoDataBuffer().addressPointer().address());
    }
}
 

/**
 * Writes the bytes for the object to the output.
 * <p>
 * This method should not be called directly, instead this serializer can be passed to {@link Kryo} write methods that accept a
 * serialier.
 *
 * @param kryo
 * @param output
 * @param object May be null if {@link #getAcceptsNull()} is true.
 */
@Override
public void write(Kryo kryo, Output output, INDArray object) {
    DataOutputStream dos = new DataOutputStream(output);
    try {
        Nd4j.write(object, dos);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    //Note: output should NOT be closed manually here - may be needed elsewhere (and closing here will cause serialization to fail)
}
 

/**
 * This method saves specified SequenceVectors model to target  OutputStream
 *
 * @param vectors SequenceVectors model
 * @param stream  Target output stream
 * @param <T>
 */
public static <T extends SequenceElement> void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                                                                    @NonNull OutputStream stream)
        throws IOException {

    InMemoryLookupTable<VocabWord> lookupTable = (InMemoryLookupTable<VocabWord>) vectors.getLookupTable();
    AbstractCache<T> vocabCache = (AbstractCache<T>) vectors.getVocab();

    try (ZipOutputStream zipfile = new ZipOutputStream(new BufferedOutputStream(new CloseShieldOutputStream(stream)));
         DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(zipfile))) {

        ZipEntry config = new ZipEntry(CONFIG_ENTRY);
        zipfile.putNextEntry(config);
        VectorsConfiguration configuration = vectors.getConfiguration();

        String json = configuration.toJson().trim();
        zipfile.write(json.getBytes("UTF-8"));

        ZipEntry vocab = new ZipEntry(VOCAB_ENTRY);
        zipfile.putNextEntry(vocab);
        zipfile.write(vocabCache.toJson().getBytes("UTF-8"));

        INDArray syn0Data = lookupTable.getSyn0();
        ZipEntry syn0 = new ZipEntry(SYN0_ENTRY);
        zipfile.putNextEntry(syn0);
        Nd4j.write(syn0Data, dos);
        dos.flush();

        INDArray syn1Data = lookupTable.getSyn1();
        if (syn1Data != null) {
            ZipEntry syn1 = new ZipEntry(SYN1_ENTRY);
            zipfile.putNextEntry(syn1);
            Nd4j.write(syn1Data, dos);
            dos.flush();
        }

        INDArray syn1NegData = lookupTable.getSyn1Neg();
        if (syn1NegData != null) {
            ZipEntry syn1neg = new ZipEntry(SYN1_NEG_ENTRY);
            zipfile.putNextEntry(syn1neg);
            Nd4j.write(syn1NegData, dos);
            dos.flush();
        }
    }
}
 

private static void writeMinMaxStats(MinMaxStats normalizerStats, DataOutputStream dos) throws IOException {
    dos.writeInt(Strategy.MIN_MAX.ordinal());
    Nd4j.write(normalizerStats.getLower(), dos);
    Nd4j.write(normalizerStats.getUpper(), dos);
}
 

@Test
public void testSerialization2() throws Exception {

    INDArray array = Nd4j.linspace(1, 5, 10);

    File tempFile = File.createTempFile("alpha", "11");
    tempFile.deleteOnExit();

    // now we serialize halfs, and we expect it to become floats on other side
    try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(tempFile.getAbsolutePath())))){
        Nd4j.write(array, dos);
    }

    // loading data back from file
    DataInputStream dis = new DataInputStream(new FileInputStream(tempFile.getAbsoluteFile()));

    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);

    INDArray exp = Nd4j.linspace(1, 5, 10);

    INDArray restored = Nd4j.read(dis);

    assertArrayEquals(exp.data().asFloat(), restored.data().asFloat(), 0.1f);
    assertEquals(DataBuffer.Type.FLOAT, exp.data().dataType());
}
 

/**
 * Write a model to an output stream
 * @param model the model to save
 * @param stream the output stream to write to
 * @param saveUpdater whether to save the updater for the model or not
 * @param dataNormalization the normalizer ot save (may be null)
 * @throws IOException
 */
public static void writeModel(@NonNull Model model, @NonNull OutputStream stream, boolean saveUpdater,DataNormalization dataNormalization)
        throws IOException {
    ZipOutputStream zipfile = new ZipOutputStream(new CloseShieldOutputStream(stream));

    // Save configuration as JSON
    String json = "";
    if (model instanceof MultiLayerNetwork) {
        json = ((MultiLayerNetwork) model).getLayerWiseConfigurations().toJson();
    } else if (model instanceof ComputationGraph) {
        json = ((ComputationGraph) model).getConfiguration().toJson();
    }

    ZipEntry config = new ZipEntry(CONFIGURATION_JSON);
    zipfile.putNextEntry(config);
    zipfile.write(json.getBytes());

    // Save parameters as binary
    ZipEntry coefficients = new ZipEntry(COEFFICIENTS_BIN);
    zipfile.putNextEntry(coefficients);
    DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(zipfile));
    INDArray params = model.params();
    if(params != null) {
        try {
            Nd4j.write(model.params(), dos);
        } finally {
            dos.flush();
        }
    } else {
        ZipEntry noParamsMarker = new ZipEntry(NO_PARAMS_MARKER);
        zipfile.putNextEntry(noParamsMarker);
    }

    if (saveUpdater) {
        INDArray updaterState = null;
        if (model instanceof MultiLayerNetwork) {
            updaterState = ((MultiLayerNetwork) model).getUpdater().getStateViewArray();
        } else if (model instanceof ComputationGraph) {
            updaterState = ((ComputationGraph) model).getUpdater().getStateViewArray();
        }

        if (updaterState != null && updaterState.length() > 0) {
            ZipEntry updater = new ZipEntry(UPDATER_BIN);
            zipfile.putNextEntry(updater);

            try {
                Nd4j.write(updaterState, dos);
            } finally {
                dos.flush();
            }
        }
    }


    if(dataNormalization != null) {
        // now, add our normalizer as additional entry
        ZipEntry nEntry = new ZipEntry(NORMALIZER_BIN);
        zipfile.putNextEntry(nEntry);
        NormalizerSerializer.getDefault().write(dataNormalization, zipfile);
    }

    dos.close();
    zipfile.close();
}
 

@Test
public void testHalfToFloat1() throws Exception {
    File tempFile = File.createTempFile("dsadasd","dsdfasd");
    tempFile.deleteOnExit();

    INDArray array = Nd4j.linspace(1, 100, 100);

    DataOutputStream stream = new DataOutputStream(new FileOutputStream(tempFile));

    Nd4j.write(array, stream);

    DataInputStream dis = new DataInputStream(new FileInputStream(tempFile));

    INDArray restoredFP16 = Nd4j.read(dis);

    //assertEquals(array, restoredFP16);


    DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT);
    assertEquals(DataBuffer.Type.FLOAT, Nd4j.dataType());
    log.error("--------------------");

    dis = new DataInputStream(new FileInputStream(tempFile));
    INDArray expFP32 = Nd4j.linspace(1, 100, 100);
    INDArray restoredFP32 = Nd4j.read(dis);

    CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext();

    assertTrue(AtomicAllocator.getInstance().getPointer(expFP32, context) instanceof FloatPointer);
    assertTrue(AtomicAllocator.getInstance().getPointer(restoredFP32, context) instanceof FloatPointer);

    assertEquals(DataBuffer.Type.FLOAT, expFP32.data().dataType());
    assertEquals(DataBuffer.Type.FLOAT, restoredFP32.data().dataType());

    assertEquals(expFP32, restoredFP32);

    DataTypeUtil.setDTypeForContext(DataBuffer.Type.HALF);
}
 

@Test
public void testAutoDecompression1() throws Exception {
    INDArray array = Nd4j.linspace(1, 250, 250);

    INDArray compressed = Nd4j.getCompressor().compress(array, "UINT8");

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    Nd4j.write(bos, compressed);

    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());

    INDArray result = Nd4j.read(bis);

    assertEquals(array, result);
}