Java Code Examples for org.nd4j.linalg.factory.Nd4j#write()
The following examples show how to use
org.nd4j.linalg.factory.Nd4j#write() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MinMaxSerializerStrategy.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Override public void write(@NonNull NormalizerMinMaxScaler normalizer, @NonNull OutputStream stream) throws IOException { try (DataOutputStream dos = new DataOutputStream(stream)) { dos.writeBoolean(normalizer.isFitLabel()); dos.writeDouble(normalizer.getTargetMin()); dos.writeDouble(normalizer.getTargetMax()); Nd4j.write(normalizer.getMin(), dos); Nd4j.write(normalizer.getMax(), dos); if (normalizer.isFitLabel()) { Nd4j.write(normalizer.getLabelMin(), dos); Nd4j.write(normalizer.getLabelMax(), dos); } dos.flush(); } }
Example 2
Source File: CudaFloatDataBufferTest.java From nd4j with Apache License 2.0 | 6 votes |
@Test public void testSerialization() throws Exception { Nd4j.getRandom().setSeed(12345); INDArray arr = Nd4j.rand(1,20); String temp = System.getProperty("java.io.tmpdir"); String outPath = FilenameUtils.concat(temp,"dl4jtestserialization.bin"); try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(outPath)))){ Nd4j.write(arr,dos); } INDArray in; try(DataInputStream dis = new DataInputStream(new FileInputStream(outPath))){ in = Nd4j.read(dis); } INDArray inDup = in.dup(); System.out.println(in); System.out.println(inDup); assertEquals(arr,in); //Passes: Original array "in" is OK, but array "inDup" is not!? assertEquals(in,inDup); //Fails }
Example 3
Source File: MiniBatchFileDataSetIterator.java From deeplearning4j with Apache License 2.0 | 6 votes |
private String[] writeData(DataSet write) throws IOException { String[] ret = new String[2]; String dataSetId = UUID.randomUUID().toString(); BufferedOutputStream dataOut = new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".bin"))); DataOutputStream dos = new DataOutputStream(dataOut); Nd4j.write(write.getFeatures(), dos); dos.flush(); dos.close(); BufferedOutputStream dataOutLabels = new BufferedOutputStream(new FileOutputStream(new File(rootDir, dataSetId + ".labels.bin"))); DataOutputStream dosLabels = new DataOutputStream(dataOutLabels); Nd4j.write(write.getLabels(), dosLabels); dosLabels.flush(); dosLabels.close(); ret[0] = new File(rootDir, dataSetId + ".bin").getAbsolutePath(); ret[1] = new File(rootDir, dataSetId + ".labels.bin").getAbsolutePath(); return ret; }
Example 4
Source File: MultiStandardizeSerializerStrategy.java From deeplearning4j with Apache License 2.0 | 6 votes |
/** * Serialize a MultiNormalizerStandardize to a output stream * * @param normalizer the normalizer * @param stream the output stream to write to * @throws IOException */ public void write(@NonNull MultiNormalizerStandardize normalizer, @NonNull OutputStream stream) throws IOException { try (DataOutputStream dos = new DataOutputStream(stream)) { dos.writeBoolean(normalizer.isFitLabel()); dos.writeInt(normalizer.numInputs()); dos.writeInt(normalizer.isFitLabel() ? normalizer.numOutputs() : -1); for (int i = 0; i < normalizer.numInputs(); i++) { Nd4j.write(normalizer.getFeatureMean(i), dos); Nd4j.write(normalizer.getFeatureStd(i), dos); } if (normalizer.isFitLabel()) { for (int i = 0; i < normalizer.numOutputs(); i++) { Nd4j.write(normalizer.getLabelMean(i), dos); Nd4j.write(normalizer.getLabelStd(i), dos); } } dos.flush(); } }
Example 5
Source File: BinarySerdeTest.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void timeOldVsNew() throws Exception { int numTrials = 1000; long oldTotal = 0; long newTotal = 0; INDArray arr = Nd4j.create(100000); Nd4j.getCompressor().compressi(arr, "GZIP"); for (int i = 0; i < numTrials; i++) { StopWatch oldStopWatch = new StopWatch(); // FIXME: int cast BufferedOutputStream bos = new BufferedOutputStream(new ByteArrayOutputStream((int) arr.length())); DataOutputStream dos = new DataOutputStream(bos); oldStopWatch.start(); Nd4j.write(arr, dos); oldStopWatch.stop(); // System.out.println("Old " + oldStopWatch.getNanoTime()); oldTotal += oldStopWatch.getNanoTime(); StopWatch newStopWatch = new StopWatch(); newStopWatch.start(); BinarySerde.toByteBuffer(arr); newStopWatch.stop(); // System.out.println("New " + newStopWatch.getNanoTime()); newTotal += newStopWatch.getNanoTime(); } oldTotal /= numTrials; newTotal /= numTrials; System.out.println("Old avg " + oldTotal + " New avg " + newTotal); }
Example 6
Source File: TestSerializationFloatToDouble.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testSerializationOnViewsNd4jWriteRead() throws Exception { int length = 100; Nd4j.create(1); DataTypeUtil.setDTypeForContext(DataType.FLOAT); INDArray arr = Nd4j.linspace(1, length, length).reshape('c', 10, 10); INDArray sub = arr.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (DataOutputStream dos = new DataOutputStream(baos)) { Nd4j.write(sub, dos); } byte[] bytes = baos.toByteArray(); //SET DATA TYPE TO DOUBLE and initialize another array with the same contents //Nd4j.create(1); DataTypeUtil.setDTypeForContext(DataType.DOUBLE); System.out.println("The data opType is " + Nd4j.dataType()); INDArray arr1 = Nd4j.linspace(1, length, length, DataType.FLOAT).reshape('c', 10, 10); INDArray sub1 = arr1.get(NDArrayIndex.interval(5, 10), NDArrayIndex.interval(5, 10)); INDArray arr2; try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes))) { arr2 = Nd4j.read(dis); } //assertEquals(sub,arr2);\ assertTrue(Transforms.abs(sub1.sub(arr2).div(sub1)).maxNumber().doubleValue() < 0.01); }
Example 7
Source File: IntegrationTestRunner.java From deeplearning4j with Apache License 2.0 | 5 votes |
public static void write(INDArray arr, File f) { try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(f)))) { Nd4j.write(arr, dos); } catch (IOException e) { throw new RuntimeException(e); } }
Example 8
Source File: NDArrayTestsFortran.java From nd4j with Apache License 2.0 | 5 votes |
@Test public void testReadWriteDouble() throws Exception { INDArray write = Nd4j.linspace(1, 4, 4); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); Nd4j.write(write, dos); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); INDArray read = Nd4j.read(dis); assertEquals(write, read); }
Example 9
Source File: NDArrayTestsFortran.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testReadWriteDouble() throws Exception { INDArray write = Nd4j.linspace(1, 4, 4, DataType.FLOAT); ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); Nd4j.write(write, dos); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); INDArray read = Nd4j.read(dis); assertEquals(write, read); }
Example 10
Source File: AeronNDArraySerdeTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void timeOldVsNew() throws Exception { int numTrials = 1000; long oldTotal = 0; long newTotal = 0; INDArray arr = Nd4j.create(100000); Nd4j.getCompressor().compressi(arr, "GZIP"); for (int i = 0; i < numTrials; i++) { StopWatch oldStopWatch = new StopWatch(); BufferedOutputStream bos = new BufferedOutputStream(new ByteArrayOutputStream((int) arr.length())); DataOutputStream dos = new DataOutputStream(bos); oldStopWatch.start(); Nd4j.write(arr, dos); oldStopWatch.stop(); // System.out.println("Old " + oldStopWatch.getNanoTime()); oldTotal += oldStopWatch.getNanoTime(); StopWatch newStopWatch = new StopWatch(); newStopWatch.start(); AeronNDArraySerde.toBuffer(arr); newStopWatch.stop(); // System.out.println("New " + newStopWatch.getNanoTime()); newTotal += newStopWatch.getNanoTime(); } oldTotal /= numTrials; newTotal /= numTrials; System.out.println("Old avg " + oldTotal + " New avg " + newTotal); }
Example 11
Source File: TestSerialization.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testSerializationFullArrayNd4jWriteRead() throws Exception { int length = 100; INDArray arrC = Nd4j.linspace(1, length, length).reshape('c', 10, 10); INDArray arrF = Nd4j.linspace(1, length, length).reshape('f', 10, 10); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (DataOutputStream dos = new DataOutputStream(baos)) { Nd4j.write(arrC, dos); } byte[] bytesC = baos.toByteArray(); baos = new ByteArrayOutputStream(); try (DataOutputStream dos = new DataOutputStream(baos)) { Nd4j.write(arrF, dos); } byte[] bytesF = baos.toByteArray(); INDArray arr2C; try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesC))) { arr2C = Nd4j.read(dis); } INDArray arr2F; try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytesF))) { arr2F = Nd4j.read(dis); } assertEquals(arrC, arr2C); assertEquals(arrF, arr2F); }
Example 12
Source File: MultiDataSet.java From deeplearning4j with Apache License 2.0 | 5 votes |
private void saveINDArrays(INDArray[] arrays, DataOutputStream dos, boolean isMask) throws IOException { if (arrays != null && arrays.length > 0) { for (INDArray fm : arrays) { if (isMask && fm == null) { INDArray temp = EMPTY_MASK_ARRAY_PLACEHOLDER.get(); if(temp == null){ EMPTY_MASK_ARRAY_PLACEHOLDER.set(Nd4j.create(new float[] {-1})); temp = EMPTY_MASK_ARRAY_PLACEHOLDER.get(); } fm = temp; } Nd4j.write(fm, dos); } } }
Example 13
Source File: WorkspaceProviderTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testWorkspacesSerde2() throws Exception { INDArray array = Nd4j.create(10).assign(1.0); INDArray restored = null; ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); Nd4j.write(array, dos); try (Nd4jWorkspace workspace = (Nd4jWorkspace) Nd4j.getWorkspaceManager() .getAndActivateWorkspace(basicConfiguration, "WS_1")) { workspace.enableDebug(true); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); DataInputStream dis = new DataInputStream(bis); restored = Nd4j.read(dis); long requiredMemory = 10 * Nd4j.sizeOfDataType(); assertEquals(requiredMemory + requiredMemory % 8, workspace.getPrimaryOffset()); assertEquals(array.length(), restored.length()); assertEquals(1.0f, restored.meanNumber().floatValue(), 1.0f); // we want to ensure it's the same cached shapeInfo used here assertEquals(array.shapeInfoDataBuffer().addressPointer().address(), restored.shapeInfoDataBuffer().addressPointer().address()); } }
Example 14
Source File: Nd4jSerializer.java From nd4j with Apache License 2.0 | 5 votes |
/** * Writes the bytes for the object to the output. * <p> * This method should not be called directly, instead this serializer can be passed to {@link Kryo} write methods that accept a * serialier. * * @param kryo * @param output * @param object May be null if {@link #getAcceptsNull()} is true. */ @Override public void write(Kryo kryo, Output output, INDArray object) { DataOutputStream dos = new DataOutputStream(output); try { Nd4j.write(object, dos); } catch (IOException e) { throw new RuntimeException(e); } //Note: output should NOT be closed manually here - may be needed elsewhere (and closing here will cause serialization to fail) }
Example 15
Source File: WordVectorSerializer.java From deeplearning4j with Apache License 2.0 | 4 votes |
/** * This method saves specified SequenceVectors model to target OutputStream * * @param vectors SequenceVectors model * @param stream Target output stream * @param <T> */ public static <T extends SequenceElement> void writeSequenceVectors(@NonNull SequenceVectors<T> vectors, @NonNull OutputStream stream) throws IOException { InMemoryLookupTable<VocabWord> lookupTable = (InMemoryLookupTable<VocabWord>) vectors.getLookupTable(); AbstractCache<T> vocabCache = (AbstractCache<T>) vectors.getVocab(); try (ZipOutputStream zipfile = new ZipOutputStream(new BufferedOutputStream(new CloseShieldOutputStream(stream))); DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(zipfile))) { ZipEntry config = new ZipEntry(CONFIG_ENTRY); zipfile.putNextEntry(config); VectorsConfiguration configuration = vectors.getConfiguration(); String json = configuration.toJson().trim(); zipfile.write(json.getBytes("UTF-8")); ZipEntry vocab = new ZipEntry(VOCAB_ENTRY); zipfile.putNextEntry(vocab); zipfile.write(vocabCache.toJson().getBytes("UTF-8")); INDArray syn0Data = lookupTable.getSyn0(); ZipEntry syn0 = new ZipEntry(SYN0_ENTRY); zipfile.putNextEntry(syn0); Nd4j.write(syn0Data, dos); dos.flush(); INDArray syn1Data = lookupTable.getSyn1(); if (syn1Data != null) { ZipEntry syn1 = new ZipEntry(SYN1_ENTRY); zipfile.putNextEntry(syn1); Nd4j.write(syn1Data, dos); dos.flush(); } INDArray syn1NegData = lookupTable.getSyn1Neg(); if (syn1NegData != null) { ZipEntry syn1neg = new ZipEntry(SYN1_NEG_ENTRY); zipfile.putNextEntry(syn1neg); Nd4j.write(syn1NegData, dos); dos.flush(); } } }
Example 16
Source File: MultiHybridSerializerStrategy.java From nd4j with Apache License 2.0 | 4 votes |
private static void writeMinMaxStats(MinMaxStats normalizerStats, DataOutputStream dos) throws IOException { dos.writeInt(Strategy.MIN_MAX.ordinal()); Nd4j.write(normalizerStats.getLower(), dos); Nd4j.write(normalizerStats.getUpper(), dos); }
Example 17
Source File: CudaHalfsTest.java From nd4j with Apache License 2.0 | 4 votes |
@Test public void testSerialization2() throws Exception { INDArray array = Nd4j.linspace(1, 5, 10); File tempFile = File.createTempFile("alpha", "11"); tempFile.deleteOnExit(); // now we serialize halfs, and we expect it to become floats on other side try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get(tempFile.getAbsolutePath())))){ Nd4j.write(array, dos); } // loading data back from file DataInputStream dis = new DataInputStream(new FileInputStream(tempFile.getAbsoluteFile())); DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT); INDArray exp = Nd4j.linspace(1, 5, 10); INDArray restored = Nd4j.read(dis); assertArrayEquals(exp.data().asFloat(), restored.data().asFloat(), 0.1f); assertEquals(DataBuffer.Type.FLOAT, exp.data().dataType()); }
Example 18
Source File: ModelSerializer.java From deeplearning4j with Apache License 2.0 | 4 votes |
/** * Write a model to an output stream * @param model the model to save * @param stream the output stream to write to * @param saveUpdater whether to save the updater for the model or not * @param dataNormalization the normalizer ot save (may be null) * @throws IOException */ public static void writeModel(@NonNull Model model, @NonNull OutputStream stream, boolean saveUpdater,DataNormalization dataNormalization) throws IOException { ZipOutputStream zipfile = new ZipOutputStream(new CloseShieldOutputStream(stream)); // Save configuration as JSON String json = ""; if (model instanceof MultiLayerNetwork) { json = ((MultiLayerNetwork) model).getLayerWiseConfigurations().toJson(); } else if (model instanceof ComputationGraph) { json = ((ComputationGraph) model).getConfiguration().toJson(); } ZipEntry config = new ZipEntry(CONFIGURATION_JSON); zipfile.putNextEntry(config); zipfile.write(json.getBytes()); // Save parameters as binary ZipEntry coefficients = new ZipEntry(COEFFICIENTS_BIN); zipfile.putNextEntry(coefficients); DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(zipfile)); INDArray params = model.params(); if(params != null) { try { Nd4j.write(model.params(), dos); } finally { dos.flush(); } } else { ZipEntry noParamsMarker = new ZipEntry(NO_PARAMS_MARKER); zipfile.putNextEntry(noParamsMarker); } if (saveUpdater) { INDArray updaterState = null; if (model instanceof MultiLayerNetwork) { updaterState = ((MultiLayerNetwork) model).getUpdater().getStateViewArray(); } else if (model instanceof ComputationGraph) { updaterState = ((ComputationGraph) model).getUpdater().getStateViewArray(); } if (updaterState != null && updaterState.length() > 0) { ZipEntry updater = new ZipEntry(UPDATER_BIN); zipfile.putNextEntry(updater); try { Nd4j.write(updaterState, dos); } finally { dos.flush(); } } } if(dataNormalization != null) { // now, add our normalizer as additional entry ZipEntry nEntry = new ZipEntry(NORMALIZER_BIN); zipfile.putNextEntry(nEntry); NormalizerSerializer.getDefault().write(dataNormalization, zipfile); } dos.close(); zipfile.close(); }
Example 19
Source File: HalfOpsTests.java From nd4j with Apache License 2.0 | 3 votes |
@Test public void testHalfToFloat1() throws Exception { File tempFile = File.createTempFile("dsadasd","dsdfasd"); tempFile.deleteOnExit(); INDArray array = Nd4j.linspace(1, 100, 100); DataOutputStream stream = new DataOutputStream(new FileOutputStream(tempFile)); Nd4j.write(array, stream); DataInputStream dis = new DataInputStream(new FileInputStream(tempFile)); INDArray restoredFP16 = Nd4j.read(dis); //assertEquals(array, restoredFP16); DataTypeUtil.setDTypeForContext(DataBuffer.Type.FLOAT); assertEquals(DataBuffer.Type.FLOAT, Nd4j.dataType()); log.error("--------------------"); dis = new DataInputStream(new FileInputStream(tempFile)); INDArray expFP32 = Nd4j.linspace(1, 100, 100); INDArray restoredFP32 = Nd4j.read(dis); CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext(); assertTrue(AtomicAllocator.getInstance().getPointer(expFP32, context) instanceof FloatPointer); assertTrue(AtomicAllocator.getInstance().getPointer(restoredFP32, context) instanceof FloatPointer); assertEquals(DataBuffer.Type.FLOAT, expFP32.data().dataType()); assertEquals(DataBuffer.Type.FLOAT, restoredFP32.data().dataType()); assertEquals(expFP32, restoredFP32); DataTypeUtil.setDTypeForContext(DataBuffer.Type.HALF); }
Example 20
Source File: CompressionSerDeTests.java From nd4j with Apache License 2.0 | 3 votes |
@Test public void testAutoDecompression1() throws Exception { INDArray array = Nd4j.linspace(1, 250, 250); INDArray compressed = Nd4j.getCompressor().compress(array, "UINT8"); ByteArrayOutputStream bos = new ByteArrayOutputStream(); Nd4j.write(bos, compressed); ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); INDArray result = Nd4j.read(bis); assertEquals(array, result); }