Java Code Examples for ucar.nc2.NetcdfFileWriter#Version

public StationSubsetWriterNetcdf(FeatureDatasetPoint fdPoint, SubsetParams ncssParams, NcssDiskCache ncssDiskCache,
    OutputStream out, NetcdfFileWriter.Version version) throws NcssException, IOException {
  super(fdPoint, ncssParams);

  this.ncssDiskCache = ncssDiskCache;
  this.out = out;
  this.version = version;

  this.netcdfResult = ncssDiskCache.getDiskCache().createUniqueFile("ncss-station", ".nc");
  List<Attribute> attribs = new ArrayList<>();
  attribs.add(new Attribute(CDM.TITLE, "Extracted data from TDS Feature Collection " + fdPoint.getLocation()));

  // get the timeUnit and altUnit from the first FeatureCollection
  assert fdPoint.getPointFeatureCollectionList().size() > 0;
  DsgFeatureCollection fc = fdPoint.getPointFeatureCollectionList().get(0);
  CalendarDateUnit timeUnit = fc.getTimeUnit();
  String altUnit = fc.getAltUnits();

  this.cfWriter = new WriterCFStationCollection(netcdfResult.getAbsolutePath(), attribs, wantedVariables, timeUnit,
      altUnit, new CFPointWriterConfig(version));
}
 

public void setOutputFilename(String filename) {
  if (filename == null)
    filename = "test";
  String location = filename;
  if (location.startsWith("file:"))
    location = location.substring(5);
  int pos = location.lastIndexOf(".");
  if (pos > 0)
    location = location.substring(0, pos);

  // change suffix
  NetcdfFileWriter.Version version = (NetcdfFileWriter.Version) netcdfVersion.getSelectedItem();
  String suffix = (version == null) ? ".nc" : version.getSuffix();
  if (filename.endsWith(".nc") && suffix.equals(".nc"))
    suffix = ".sub.nc";
  location += suffix;

  outputFilename.setText(location);
}
 

public long makeFile(String location, ucar.nc2.dt.GridDataset gds, List<String> gridList, LatLonRect llbb,
    int horizStride, Range zRange, CalendarDateRange dateRange, int stride_time, boolean addLatLon,
    NetcdfFileWriter.Version version) throws IOException, InvalidRangeException {

  return makeOrTestSize(location, gds, gridList, llbb, horizStride, zRange, dateRange, stride_time, addLatLon, false,
      version);
}
 

double rewrite(String filenameIn, String filenameOut, NetcdfFileWriter.Version version,
    Nc4Chunking.Strategy chunkerType, int deflateLevel, boolean shuffle, Formatter fw) throws IOException {

  ucar.nc2.dt.GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(filenameIn);
  Nc4Chunking chunking = (version == NetcdfFileWriter.Version.netcdf3) ? null
      : Nc4ChunkingStrategy.factory(chunkerType, deflateLevel, shuffle);
  NetcdfFileWriter writer = NetcdfFileWriter.createNew(version, filenameOut, chunking);

  long start = System.currentTimeMillis();

  long totalBytes;
  try {
    totalBytes = CFGridWriter2.writeFile(gds, null, null, null, 1, null, null, 1, true, writer);
  } catch (Throwable e) {
    e.printStackTrace();
    return 0;
  }

  File fin = new File(filenameIn);
  double lenIn = (double) fin.length();
  lenIn /= 1000 * 1000;

  File fout = new File(filenameOut);
  double lenOut = (double) fout.length();
  lenOut /= 1000 * 1000;

  System.out.format("   %10.3f: %s%n", lenOut / lenIn, fout.getCanonicalPath());
  double took = (System.currentTimeMillis() - start) / 1000.0;
  System.out.format("   that took: %f secs%n", took);

  if (fw != null)
    fw.format("%s,%10.3f, %s,%s, %d, %10.3f,%10.3f,%d%n", fin.getName(), lenIn,
        (chunkerType != null) ? chunkerType : "nc3", shuffle ? "shuffle" : "", deflateLevel, lenOut, lenOut / lenIn,
        totalBytes);

  return lenOut;
}
 

public static NetcdfFileFormat convertToNetcdfFileFormat(NetcdfFileWriter.Version version) {
  switch (version) {
    case netcdf3:
      return NetcdfFileFormat.NETCDF3;
    case netcdf4:
      return NetcdfFileFormat.NETCDF4;
    case netcdf4_classic:
      return NetcdfFileFormat.NETCDF4_CLASSIC;
    case netcdf3c64:
      return NetcdfFileFormat.NETCDF3_64BIT_OFFSET;
    default:
      throw new IllegalStateException("Unsupported version: " + version);
  }
}
 

public static NetcdfFileWriter.Version convertToNetcdfFileWriterVersion(NetcdfFileFormat format) {
  switch (format) {
    case NETCDF3:
      return NetcdfFileWriter.Version.netcdf3;
    case NETCDF4:
      return NetcdfFileWriter.Version.netcdf4;
    case NETCDF4_CLASSIC:
      return NetcdfFileWriter.Version.netcdf4_classic;
    case NETCDF3_64BIT_OFFSET:
      return NetcdfFileWriter.Version.netcdf3c64;
    default:
      throw new IllegalStateException("Unsupported format: " + format);
  }
}
 

public long makeFile(String location, ucar.nc2.dt.GridDataset gds, List<String> gridList, ProjectionRect llbb,
    int horizStride, Range zRange, CalendarDateRange dateRange, int stride_time, boolean addLatLon,
    NetcdfFileWriter.Version version) throws IOException, InvalidRangeException {

  return makeOrTestSize(location, gds, gridList, llbb, horizStride, zRange, dateRange, stride_time, addLatLon, false,
      version);
}
 

public NetcdfFileWriter.Version getVersion() {
  return NetcdfFormatWriter.convertToNetcdfFileWriterVersion(format);
}
 

private NetcdfFormatWriter(Builder builder) throws IOException {
  this.location = builder.location;
  this.format = builder.format;
  this.isNewFile = builder.isNewFile;
  this.fill = builder.fill;
  this.extraHeaderBytes = builder.extraHeaderBytes;
  this.preallocateSize = builder.preallocateSize;
  this.chunker = builder.chunker;
  this.useJna = builder.useJna || format.isNetdf4format();

  this.ncout = NetcdfFile.builder().setRootGroup(builder.rootGroup).build();
  this.rootGroup = this.ncout.getRootGroup();

  if (!isNewFile) {
    existingRaf = new ucar.unidata.io.RandomAccessFile(location, "rw");
    NetcdfFileFormat existingVersion = NetcdfFileFormat.findNetcdfFormatType(existingRaf);
    if (format != null && format != existingVersion) {
      existingRaf.close();
      throw new IllegalArgumentException("Existing file at location" + location + " (" + existingVersion
          + ") does not match requested version " + format);
    }
  } else {
    existingRaf = null;
  }

  if (useJna) {
    String className = "ucar.nc2.jni.netcdf.Nc4Iosp";
    IOServiceProviderWriter spi;
    try {
      Class iospClass = this.getClass().getClassLoader().loadClass(className);
      NetcdfFileWriter.Version version = convertToNetcdfFileWriterVersion(format);
      Constructor<IOServiceProviderWriter> ctor = iospClass.getConstructor(version.getClass());
      spi = ctor.newInstance(version);

      Method method = iospClass.getMethod("setChunker", Nc4Chunking.class);
      method.invoke(spi, chunker);
    } catch (Throwable e) {
      throw new IllegalArgumentException(className + " cannot use JNI/C library err= " + e.getMessage());
    }
    spiw = spi;
  } else {
    spiw = new N3iospWriter(builder.getIosp());
  }

  // If anything fails, make sure that resources are closed.
  try {
    if (!isNewFile) {
      spiw.openForWriting(existingRaf, this.ncout, null);
    } else {
      spiw.create(location, this.ncout, extraHeaderBytes, preallocateSize, testIfLargeFile());
    }
    spiw.setFill(fill);
  } catch (Throwable t) {
    spiw.close();
    throw t;
  }
}
 

public CFPointWriterConfig(NetcdfFileWriter.Version version, Nc4Chunking chunking) {
  this.version = version;
  this.chunking = chunking;
}
 

public CFPointWriterConfig(NetcdfFileWriter.Version version) {
  this(version, new Nc4ChunkingDefault()); // The default chunker used in Nc4Iosp.
}
 

private void writeDataset(String location, FeatureType ftype, NetcdfFileWriter.Version version, int countExpected)
    throws IOException {
  File fileIn = new File(location);
  long start = System.currentTimeMillis();
  outOrg = tempFolder.newFile().getPath();
  outNew = tempFolder.newFile().getPath();

  System.out.printf("================ TestCFPointWriter%n read %s size=%d%n write to=%s%n", fileIn.getAbsoluteFile(),
      fileIn.length(), outOrg);

  // open point dataset
  Formatter out = new Formatter();
  try (FeatureDataset fdataset = FeatureDatasetFactoryManager.open(ftype, location, null, out)) {
    if (fdataset == null) {
      System.out.printf("**failed on %s %n --> %s %n", location, out);
      assert false;
    }
    assert fdataset instanceof FeatureDatasetPoint;
    FeatureDatasetPoint fdpoint = (FeatureDatasetPoint) fdataset;

    CFPointWriterConfig configOrg = new CFPointWriterConfig(version);
    int count = CFPointWriter.writeFeatureCollection(fdpoint, outOrg, configOrg);
    long took = System.currentTimeMillis() - start;
    System.out.printf(" CFPointWriter nrecords written = %d took=%d msecs%n%n", count, took);
    if (countExpected > 0) {
      assert count == countExpected : "count =" + count + " expected " + countExpected;
    }
  }

  try (FeatureDataset fdataset = FeatureDatasetFactoryManager.open(ftype, location, null, out)) {
    FeatureDatasetPoint fdpoint = (FeatureDatasetPoint) fdataset;
    ucar.nc2.ft.point.writer2.CFPointWriterConfig configNew = ucar.nc2.ft.point.writer2.CFPointWriterConfig.builder()
        .setFormat(NetcdfFormatWriter.convertToNetcdfFileFormat(version)).build();
    int countNew = ucar.nc2.ft.point.writer2.CFPointWriter.writeFeatureCollection(fdpoint, outNew, configNew);
    long tookNew = System.currentTimeMillis() - start;
    System.out.printf(" CFPointWriterNew nrecords written = %d took=%d msecs%n%n", countNew, tookNew);
    if (countExpected > 0) {
      assert countNew == countExpected : "count =" + countNew + " expected " + countExpected;
    }

    compare(outOrg, outNew);
  }
}
 

private void writeDataset(String location, FeatureType ftype, NetcdfFileWriter.Version version, int countExpected)
    throws IOException {
  File fileIn = new File(location);
  File fileOrg = tempFolder.newFile();
  long start = System.currentTimeMillis();

  System.out.printf("================ TestCFPointWriter%n read %s size=%d%n write to=%s%n", fileIn.getAbsoluteFile(),
      fileIn.length(), fileOrg.getAbsoluteFile());

  // open point dataset
  Formatter out = new Formatter();
  try (FeatureDataset fdataset = FeatureDatasetFactoryManager.open(ftype, location, null, out)) {
    if (fdataset == null) {
      System.out.printf("**failed on %s %n --> %s %n", location, out);
      assert false;
    }
    assert fdataset instanceof FeatureDatasetPoint;
    FeatureDatasetPoint fdpoint = (FeatureDatasetPoint) fdataset;

    CFPointWriterConfig configOrg = new CFPointWriterConfig(version);
    int count = CFPointWriter.writeFeatureCollection(fdpoint, fileOrg.getPath(), configOrg);
    long took = System.currentTimeMillis() - start;
    System.out.printf(" CFPointWriter nrecords written = %d took=%d msecs %s%n", count, took,
        fdpoint.getFeatureType());
    assert count == countExpected : "count =" + count + " expected " + countExpected;
  }

  try (FeatureDataset fdataset = FeatureDatasetFactoryManager.open(ftype, location, null, out)) {
    FeatureDatasetPoint fdpoint = (FeatureDatasetPoint) fdataset;
    File fileNew = tempFolder.newFile();
    ucar.nc2.ft.point.writer2.CFPointWriterConfig configNew = ucar.nc2.ft.point.writer2.CFPointWriterConfig.builder()
        .setFormat(NetcdfFormatWriter.convertToNetcdfFileFormat(version)).build();
    int countNew =
        ucar.nc2.ft.point.writer2.CFPointWriter.writeFeatureCollection(fdpoint, fileNew.getPath(), configNew);
    long tookNew = System.currentTimeMillis() - start;
    System.out.printf(" CFPointWriterNew nrecords written = %d took=%d msecs%n", countNew, tookNew);
    assert countNew == countExpected : "count =" + countNew + " expected " + countExpected;

    compare(fileOrg.getPath(), fileNew.getPath());
  }
}
 

@Test
public void writeTestFile() throws IOException, InvalidRangeException {
  // skip test requiring netcdf4 if not present.
  NetcdfFileWriter.Version version = NetcdfFormatWriter.convertToNetcdfFileWriterVersion(format);
  if (version.useJniIosp() && !Nc4Iosp.isClibraryPresent()) {
    return;
  }
  System.out.printf("Test Dataset %s type %s%n", endpoint, type);
  File tempFile = tempFolder.newFile();
  File tempFile2 = tempFolder.newFile();

  try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) {
    Assert.assertNotNull(endpoint, cc);
    CoverageCollection gcs = cc.findCoverageDataset(type);

    for (String covName : covList) {
      Assert.assertNotNull(covName, gcs.findCoverage(covName));
    }

    // write the file using CFGridCoverageWriter2
    System.out.printf(" CFGridCoverageWriter2 write to %s%n", tempFile2.getAbsolutePath());

    try (NetcdfFileWriter writer = NetcdfFileWriter
        .createNew(NetcdfFormatWriter.convertToNetcdfFileWriterVersion(format), tempFile2.getPath(), null)) {
      Optional<Long> estimatedSizeo = CFGridCoverageWriter2.write(gcs, covList, params, false, writer);
      if (!estimatedSizeo.isPresent()) {
        throw new InvalidRangeException("Request contains no data: " + estimatedSizeo.getErrorMessage());
      }
    }

    // write the file using CFGridCoverageWriter
    System.out.printf(" CFGridCoverageWriter write to %s%n", tempFile.getAbsolutePath());
    NetcdfFormatWriter.Builder writerb =
        NetcdfFormatWriter.builder().setNewFile(true).setFormat(format).setLocation(tempFile.getPath());
    NetcdfFormatWriter.Result result = CFGridCoverageWriter.write(gcs, covList, params, false, writerb, 0);
    if (!result.wasWritten()) {
      throw new InvalidRangeException("Error writing: " + result.getErrorMessage());
    }
  }

  // Compare that the files are identical
  try (NetcdfFile org = NetcdfFile.open(tempFile2.getPath())) {
    try (NetcdfFile copy = NetcdfFiles.open(tempFile.getPath())) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      boolean ok = compare.compare(org, copy, new FileWritingObjFilter());
      System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f);
      assertThat(ok).isTrue();
    }
  }
}
 

double writeNetcdf(String fileInName, NetcdfFileWriter.Version version, Nc4Chunking.Strategy chunkerType,
    int deflateLevel, boolean shuffle) throws IOException {

  File fin = new File(fileInName);
  // if (fin.length() > 2000 * 1000 * 1000) {
  // System.out.format(" skip %s: %d%n", fin.getName(), fin.length());
  // return 0; // skip > 2G
  // }
  Formatter foutf = new Formatter();
  foutf.format("%s", fin.getName());
  if (deflateLevel > 0)
    foutf.format(".%d", deflateLevel);
  if (chunkerType != null)
    foutf.format(".%s", chunkerType);
  if (shuffle)
    foutf.format(".shuffle");
  foutf.format("%s", version.getSuffix());

  File fout = new File(dirOut, foutf.toString());

  ucar.nc2.dt.GridDataset gds = ucar.nc2.dt.grid.GridDataset.open(fileInName);
  Nc4Chunking chunking = (version == NetcdfFileWriter.Version.netcdf3) ? null
      : Nc4ChunkingStrategy.factory(chunkerType, deflateLevel, shuffle);
  NetcdfFileWriter writer = NetcdfFileWriter.createNew(version, fout.getCanonicalPath(), chunking);

  long start = System.currentTimeMillis();

  long totalBytes;
  try {
    totalBytes = CFGridWriter2.writeFile(gds, null, null, null, 1, null, null, 1, false, writer);
    totalBytes /= 1000 * 1000;
  } catch (Throwable e) {
    e.printStackTrace();
    return 0;
  }

  double lenIn = (double) fin.length();
  lenIn /= 1000 * 1000;

  File fout2 = new File(dirOut, foutf.toString());
  double lenOut = (double) fout2.length();
  lenOut /= 1000 * 1000;

  System.out.format("   %10.3f: %s%n", lenOut / lenIn, fout.getCanonicalPath());
  double took = (System.currentTimeMillis() - start) / 1000.0;
  System.out.format("   that took: %f secs%n", took);

  fw.printf("%s,%10.3f, %s,%s, %d, %10.3f,%10.3f,%d,%f,%f%n", fin.getName(), lenIn,
      (chunkerType != null) ? chunkerType : "nc3", shuffle ? "shuffle" : "", deflateLevel, lenOut, lenOut / lenIn,
      totalBytes, totalBytes / lenOut, took);

  return lenOut;
}
 

public static void makeFileVersioned(String location, ucar.nc2.dt.GridDataset gds, List<String> gridList,
    LatLonRect llbb, CalendarDateRange dateRange, NetcdfFileWriter.Version version)
    throws IOException, InvalidRangeException {
  CFGridWriter writer = new CFGridWriter();
  writer.makeOrTestSize(location, gds, gridList, llbb, 1, null, dateRange, 1, false, false, version);
}
 

public static String getFileNameForResponse(String pathInfo, NetcdfFileWriter.Version version) {
  Preconditions.checkNotNull(version, "version == null");
  return getFileNameForResponse(pathInfo, version.getSuffix());
}
 

/**
 * Read an NcML and write an equivilent NetcdfFile to a physical file, using Netcdf-3 file format.
 * The NcML may have a referenced dataset in the location URL, in which case the underlying data
 * (modified by the NcML) is written to the new file. If the NcML does not have a referenced dataset,
 * then the new file is filled with fill values, like ncgen.
 *
 * @param ncml read NcML from this input stream
 * @param fileOutName write to this local file
 * @param version kind of netcdf file
 * @param chunker optional chunking (netcdf4 only)
 */
public static void writeNcMLToFile(InputStream ncml, String fileOutName, NetcdfFileWriter.Version version,
    Nc4Chunking chunker) throws IOException {
  NetcdfDataset ncd = NcMLReader.readNcML(ncml, null);
  FileWriter2 writer = new FileWriter2(ncd, fileOutName, version, chunker);
  NetcdfFile result = writer.write();
  result.close();
  ncd.close();
}