ucar.ma2.InvalidRangeException Java Examples

/**
 * Write String data to a CHAR variable.
 *
 * @param v variable to write to
 * @param origin offset to start writing, ignore the strlen dimension.
 * @param values write this array; must be ArrayObject of String
 * @throws IOException if I/O error
 * @throws InvalidRangeException if values Array has illegal shape
 */
public void writeStringDataToChar(Variable v, int[] origin, Array values) throws IOException, InvalidRangeException {
  if (values.getElementType() != String.class)
    throw new IllegalArgumentException("values must be an ArrayObject of String ");

  if (v.getDataType() != DataType.CHAR)
    throw new IllegalArgumentException("variable " + v.getFullName() + " is not type CHAR");

  int rank = v.getRank();
  int strlen = v.getShape(rank - 1);

  // turn it into an ArrayChar
  ArrayChar cvalues = ArrayChar.makeFromStringArray((ArrayObject) values, strlen);

  int[] corigin = new int[rank];
  System.arraycopy(origin, 0, corigin, 0, rank - 1);

  write(v, corigin, cvalues);
}
 

/**
 * Read a section of the local Variable.
 *
 * @param mainv aggregated Variable
 * @param cancelTask let user cancel
 * @param section reletive to the local Variable
 * @return the complete Array for mainv
 * @throws IOException on I/O error
 * @throws InvalidRangeException on section error
 */
protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section)
    throws IOException, InvalidRangeException {
  NetcdfFile ncd = null;
  try {
    ncd = acquireFile(cancelTask);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;

    Variable v = findVariable(ncd, mainv);
    if (debugRead) {
      Section want = new Section(section);
      System.out.printf("Agg.read(%s) %s from %s in %s%n", want, mainv.getNameAndDimensions(),
          v.getNameAndDimensions(), getLocation());
    }

    return v.read(section);

  } finally {
    close(ncd);
  }
}
 

@Test
public void testUpdateLastModified() throws IOException, InvalidRangeException, InterruptedException {
  // make sure that the extra file is not in the agg
  move(extraFile);

  // open the agg
  NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null);
  long start = ncfile.getLastModified();

  // now make sure that the extra file is in the agg
  moveBack(extraFile);

  // reread
  long end = ncfile.getLastModified();
  assert (end > start);

  // again
  long end2 = ncfile.getLastModified();
  assert (end == end2);

  ncfile.close();
}
 

@Category(NeedsCdmUnitTest.class)
public void testGroupAndMembers() throws InvalidRangeException, IOException {
  NetcdfFile ncfile = NetcdfFiles.open(TestDir.cdmUnitTestDir + "formats/netcdf4/compound/simple_nc4.nc4");
  Variable v = ncfile.findVariable("grp1/data");
  assert v != null;

  ParsedSectionSpec spec = ParsedSectionSpec.parseVariableSection(ncfile, "grp1/data");
  System.out.printf("%s%n", spec);
  assert spec.section.equals(v.getShapeAsSection());

  spec = ParsedSectionSpec.parseVariableSection(ncfile, "grp2/data.i1");
  System.out.printf("%s%n", spec);

  Variable s = ncfile.findVariable("grp2/data");
  assert spec.section.equals(s.getShapeAsSection());

  v = ncfile.findVariable("grp2/data.i1");
  assert spec.child.section.equals(v.getShapeAsSection());

  ncfile.close();
}
 

Array readArray() throws IOException {
  int[] shape = mds.dimLength;
  DataType dataType = typeInfo.dataType;
  Layout layout;
  try {
    if (isChunked) {
      layout = new H5tiledLayout(this, dataType, new Section(shape));
    } else {
      layout = new LayoutRegular(dataPos, dataType.getSize(), shape, null);
    }
  } catch (InvalidRangeException e) {
    // cant happen because we use null for wantSection
    throw new IllegalStateException();
  }
  Object data = IospHelper.readDataFill(raf, layout, dataType, getFillValue(), typeInfo.endian, false);
  return Array.factory(dataType, shape, data);
}
 

@Test
public void testReadSubsection() throws IOException, InvalidRangeException {
  String location = TestDir.cdmUnitTestDir + "formats/netcdf4/ncom_relo_fukushima_1km_tmp_2011040800_t000.nc4";
  try (NetcdfFile ncfile = NetcdfFiles.open(location); NetcdfFile jni = openJni(location)) {

    jni.setLocation(location + " (jni)");

    // float salinity(time=1, depth=40, lat=667, lon=622);
    Array data1 = read(ncfile, "salinity", "0,11:12,22,:");
    // NCdumpW.printArray(data1);
    System.out.printf("Read from jni%n");
    Array data2 = read(jni, "salinity", "0,11:12,22,:");
    assert MAMath.nearlyEquals(data1, data2);
    System.out.printf("data is equal%n");
  }
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInFilenameOrNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

private StructureData nextStructureData() {
  StructureDataW sdata = new StructureDataW(members);

  for (Variable var : vars) {
    Section.Builder sb = Section.builder();
    try {
      sb.appendRange(outerIndex, outerIndex);
      sb.appendRange(count, count);
      for (int i = 2; i < var.getRank(); i++)
        sb.appendRangeAll();
      Array data = var.read(sb.build());
      sdata.setMemberData(var.getShortName(), data);

    } catch (InvalidRangeException | IOException e) {
      throw new RuntimeException(e);
    }

  }

  return sdata;
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "file:./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDataset.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

@Test
public void test1() throws IOException, InvalidRangeException {
  String filename = "file:./" + TestNcMLRead.topDir + "aggSynthetic.xml";
  NetcdfFile ncfile = NcMLReaderNew.readNcML(filename, null, null).build();

  Variable v = ncfile.findVariable("time");
  assert v != null;
  String testAtt = v.findAttributeString("units", null);
  assert testAtt != null;
  assert testAtt.equals("months since 2000-6-16 6:00");

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile, "T");
  testReadSlice(ncfile, "T");

  ncfile.close();
}
 

public void testReadSlice(NetcdfFile ncfile, int[] origin, int[] shape) throws IOException, InvalidRangeException {

    Variable v = ncfile.findVariable("T");

    Array data = v.read(origin, shape);
    assert data.getRank() == 3;
    assert data.getSize() == shape[0] * shape[1] * shape[2];
    assert data.getShape()[0] == shape[0] : data.getShape()[0] + " " + shape[0];
    assert data.getShape()[1] == shape[1];
    assert data.getShape()[2] == shape[2];
    assert data.getElementType() == double.class;

    Index tIndex = data.getIndex();
    for (int i = 0; i < shape[0]; i++)
      for (int j = 0; j < shape[1]; j++)
        for (int k = 0; k < shape[2]; k++) {
          double val = data.getDouble(tIndex.set(i, j, k));
          Assert2.assertNearlyEquals(val, 100 * (i + origin[0]) + 10 * j + k);
        }

  }
 

@Test
public void testRewriteHeader3() throws IOException, InvalidRangeException {
  String filename = tempFolder.newFile().getAbsolutePath();

  try (NetcdfFileWriter file = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, filename)) {
    file.addGlobalAttribute("att8", "1234567890");
    file.setExtraHeaderBytes(10);
    file.create();

    file.setRedefineMode(true);
    file.addGlobalAttribute("att8", "123456789012345");
    boolean rewriteAll = file.setRedefineMode(false);
    assert !rewriteAll;

    Attribute att = file.findGlobalAttribute("att8");
    assert att != null;
    assert att.getStringValue().equals("123456789012345") : att.getStringValue();
  }
}
 

static void testAllInDir(File dir, MClosure closure) throws IOException, InvalidRangeException {
  File[] fa = dir.listFiles();
  if (fa == null || fa.length == 0)
    return;

  List<File> list = Arrays.asList(fa);
  Collections.sort(list);

  for (File f : list) {
    if (f.isDirectory())
      testAllInDir(f, closure);
    else {
      closure.run(f.getPath());
    }
  }
}
 

static int timeDataRead(String remote, int stride) throws IOException, InvalidRangeException {
  logger.debug("--CdmRemote Read {} stride={}", remote, stride);

  try (NetcdfFile ncremote = new CdmRemote(remote)) {

    String gridName = "10_metre_V_wind_component_surface";
    Variable vs = ncremote.findVariable(gridName);
    Assert.assertNotNull(gridName, vs);

    Section section = vs.getShapeAsSection();
    Assert.assertEquals(3, section.getRank());
    Section want = Section.builder().appendRange(1).appendRange(section.getRange(1).copyWithStride(stride))
        .appendRange(section.getRange(2).copyWithStride(stride)).build();

    long start = System.currentTimeMillis();

    Array data = vs.read(want);

    long took = System.currentTimeMillis() - start;
    logger.debug("took={} size={}", took, data.getSize());

  }
  return 1;
}
 

public static ucar.nc2.util.Optional<Long> getSizeOfOutput(CoverageCollection gdsOrg, List<String> gridNames,
    SubsetParams subset, boolean tryToAddLatLon2D) throws IOException, InvalidRangeException {
  CFGridCoverageWriter2 writer2 = new CFGridCoverageWriter2();
  // null location. It's ok; we'll never write the file.
  NetcdfFileWriter writer = NetcdfFileWriter.createNew(null, false);
  // COVERITY[RESOURCE_LEAK]
  return writer2.writeFile(gdsOrg, gridNames, subset, tryToAddLatLon2D, true, writer);
}
 

@Test
public void checkPaddingOnWriteReadOneDimShortArrayOnlyRecordVar() throws IOException, InvalidRangeException {
  File tmpDataDir = tempFolder.newFolder();
  File testFile = new File(tmpDataDir, "file.nc");

  NetcdfFormatWriter.Builder writerb = NetcdfFormatWriter.createNewNetcdf3(testFile.getPath());
  writerb.addDimension(Dimension.builder().setName("v").setIsUnlimited(true).build());

  Variable.Builder v = writerb.addVariable("v", DataType.SHORT, "v");
  assertEquals(2, v.getElementSize());

  short[] data = {1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -2, -3, -4, -5, -6, -7, -8, -9};

  try (NetcdfFormatWriter ncfWriteable = writerb.build()) {
    Variable var = ncfWriteable.findVariable("v");
    Array dataArray = Array.factory(DataType.SHORT, new int[] {data.length}, data);
    ncfWriteable.write(var.getFullNameEscaped(), dataArray);
  }

  try (NetcdfFile ncf = NetcdfFiles.open(testFile.getPath())) {
    Variable readVar = ncf.findVariable("v");
    assertEquals(readVar.getDataType(), DataType.SHORT);
    assertEquals(2, readVar.getElementSize());

    int[] org = {0};
    short[] readdata = (short[]) readVar.read(org, readVar.getShape()).copyTo1DJavaArray();

    assertArrayEquals(data, readdata);
  }
}
 

@Test
public void testWrf() throws IOException, InvalidRangeException {
  String filename = TestDir.cdmUnitTestDir + "conventions/wrf/global.nc";
  test(filename, "z", "T", "Time", VerticalCT.Type.WRFEta, WRFEta.class, SimpleUnit.pressureUnit);
  test(filename, "z", "U", "Time", VerticalCT.Type.WRFEta, WRFEta.class, SimpleUnit.pressureUnit);
  test(filename, "z", "V", "Time", VerticalCT.Type.WRFEta, WRFEta.class, SimpleUnit.pressureUnit);
  test(filename, "z_stag", "W", "Time", VerticalCT.Type.WRFEta, WRFEta.class, SimpleUnit.meterUnit);
}
 

static void doAll(String url, Stat tstat1, Stat tstat2) throws IOException, InvalidRangeException {
  // testCdmremoteCompress(url, stat1.setName("compress"), stat2.setName("nocomprs"), 10, true);
  System.out.printf("%n------ filename %s%n", url);

  Stat stat1 = new Stat(tstat1.getName(), false);
  Stat stat2 = new Stat(tstat2.getName(), false);

  compare(url, stat1, stat2, 5, true);

  System.out.printf(" %s MB/sec%n", stat1);
  System.out.printf(" %s MB/sec%n", stat2);

  tstat1.add(stat1);
  tstat2.add(stat2);
}
 

@Test
public void testNcmlDirect() throws IOException, InvalidRangeException {
  String filename = "file:./" + TestNcMLRead.topDir + "aggExisting.xml";

  NetcdfFile ncfile = NcMLReaderNew.readNcML(filename, null, null).build();
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

static void testOpenDataset(String dir, final String suffix) throws IOException, InvalidRangeException {
  final Average fileAvg = new Average();
  //
  testAllInDir(new File(dir), new MClosure() {
    public void run(String filename) throws IOException, InvalidRangeException {
      if (!filename.endsWith(suffix))
        return;
      // System.out.println(" open "+suffix+" file ="+filename);
      openFile(filename, fileAvg, true);
    }
  });
  System.out.println("*** open " + suffix + " datasets  =" + fileAvg);
}
 

@Test
public void testBestPresent() throws IOException, InvalidRangeException {
  String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4";
  String covName = "Temperature_altitude_above_msl";

  logger.debug("testBestPresent Dataset {} coverage {}", endpoint, covName);

  try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) {
    Assert.assertNotNull(endpoint, cc);
    CoverageCollection gcs = cc.findCoverageDataset(FeatureType.GRID);
    Assert.assertNotNull("gcs", gcs);
    Coverage cover = gcs.findCoverage(covName);
    Assert.assertNotNull(covName, cover);

    SubsetParams params = new SubsetParams();
    params.set(SubsetParams.timePresent, true);
    params.setVertCoord(3658.0);
    logger.debug("  subset {}", params);

    GeoReferencedArray geo = cover.readData(params);

    // should not be missing !
    assert !Float.isNaN(geo.getData().getFloat(0));

    Array data = geo.getData();
    Index ai = data.getIndex();
    float testValue = data.getFloat(ai.set(0, 0, 3, 0));
    Assert2.assertNearlyEquals(244.8f, testValue);
  }
}
 

private Array doLimitedRead(Variable v) throws IOException, InvalidRangeException {
  long size = v.getSize() * v.getElementSize();
  if (size < 1000 * 1000 || v.getRank() < 3) {
    if (showDetail)
      System.out.printf(" thread %s read %s bytes = %d ", who, v.getFullName(), size);
    return v.read();

  } else {
    // randomly choose a 2D slice
    int rank = v.getRank();
    List<Range> ranges = new ArrayList<>();
    int i = 0;
    for (Dimension dim : v.getDimensions()) {
      if (i < rank - 2) {
        int first = random.nextInt(dim.getLength());
        ranges.add(new Range(first, first));
      } else {
        ranges.add(new Range(0, dim.getLength() - 1));
      }
      i++;
    }
    Section s = new Section(ranges);
    if (showDetail)
      System.out.printf(" thread %s read %s(%s) bytes= %d ", who, v.getFullName(), s, s.computeSize());
    Array result = v.read(s);
    assert result.getSize() == s.computeSize();
    return result;
  }
}
 

@Test // there is no interval with offset value = 51
public void testNoIntervalFound() throws IOException, InvalidRangeException {
  String endpoint = TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4";
  String covName = "Momentum_flux_u-component_surface_Mixed_intervals_Average";

  logger.debug("test1Runtime1TimeOffset Dataset {} coverage {}", endpoint, covName);

  try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) {
    Assert.assertNotNull(endpoint, cc);
    CoverageCollection gcs = cc.findCoverageDataset(FeatureType.FMRC);
    Assert.assertNotNull("gcs", gcs);
    Coverage cover = gcs.findCoverage(covName);
    Assert.assertNotNull(covName, cover);

    SubsetParams params = new SubsetParams();
    CalendarDate runtime = CalendarDate.parseISOformat(null, "2015-03-01T12:00:00Z");
    params.set(SubsetParams.runtime, runtime);
    double offsetVal = 51.0; // should fail
    params.set(SubsetParams.timeOffset, offsetVal);
    logger.debug("  subset {}", params);

    GeoReferencedArray geo = cover.readData(params);
    testGeoArray(geo, runtime, null, offsetVal);

    // should be empty, but instead its a bunch of NaNs
    assert Float.isNaN(geo.getData().getFloat(0));
  }
}
 

@Test
public void testCacheIsUsed() throws IOException, InvalidRangeException {
  String filename = "file:TestAggExistingCache.xml";
  System.out.printf("%s%n", filename);

  String cacheDirName = tempFolder.newFolder().getAbsolutePath() + "/";
  cacheDirName = StringUtil2.replace(cacheDirName, '\\', "/"); // no nasty backslash
  System.out.printf("cacheDir=%s%n", cacheDirName);
  File cacheDir = new File(cacheDirName);
  FileUtils.deleteDirectory(cacheDir); // from commons-io
  assert !cacheDir.exists();

  DiskCache2 cache = new DiskCache2(cacheDirName, false, 0, 0);
  cache.setAlwaysUseCache(true);
  Assert.assertEquals(cacheDirName, cache.getRootDirectory());
  assert new File(cache.getRootDirectory()).exists();

  Aggregation.setPersistenceCache(cache);
  AggregationExisting.countCacheUse = 0;

  try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) {
    System.out.println(" TestNcmlAggExisting.open " + filename);
    Array ATssta = ncfile.readSection("ATssta(:,0,0,0)");
    Assert.assertEquals(4, ATssta.getSize());
  }
  Assert.assertEquals(0, AggregationExisting.countCacheUse);
  AggregationExisting.countCacheUse = 0;

  try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) {
    System.out.println(" TestNcmlAggExisting.open " + filename);
    Array ATssta = ncfile.readSection("ATssta(:,0,0,0)");
    Assert.assertEquals(4, ATssta.getSize());
  }
  Assert.assertEquals(8, AggregationExisting.countCacheUse);
}
 

@Test
public void testSrcTimePresent() throws IOException, InvalidRangeException {
  String endpoint = TestDir.cdmUnitTestDir + "ncss/GFS/CONUS_80km/GFS_CONUS_80km_20120227_0000.grib1";
  String covName = "Temperature_isobaric";

  logger.debug("testSrcTimePresent Dataset {} coverage {}", endpoint, covName);

  try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) {
    Assert.assertNotNull(endpoint, cc);
    CoverageCollection gcs = cc.findCoverageDataset(FeatureType.GRID);
    Assert.assertNotNull("gcs", gcs);
    Coverage cover = gcs.findCoverage(covName);
    Assert.assertNotNull(covName, cover);

    CoverageCoordSys cs = cover.getCoordSys();
    CoverageCoordAxis timeAxis = cs.getAxis(AxisType.Time);
    Assert.assertNotNull("timeoffset axis", timeAxis);
    Assert.assertEquals(36, timeAxis.getNcoords());

    SubsetParams params = new SubsetParams();
    params.set(SubsetParams.timePresent, true);
    logger.debug("  subset {}", params);
    GeoReferencedArray geo = cover.readData(params);

    int[] resultShape = geo.getData().getShape();
    int[] expectShape = new int[] {1, 29, 65, 93};
    Assert.assertArrayEquals("shape", expectShape, resultShape);

    CoverageCoordSys geocs = geo.getCoordSysForData();
    CoverageCoordAxis toAxis2 = geocs.getAxis(AxisType.Time);
    Assert.assertNotNull("timeoffset axis", toAxis2);
    Assert.assertEquals(1, toAxis2.getNcoords());
  }
}
 

@Test
public void testNcmlDatasetWcoords() throws IOException, InvalidRangeException {
  String filename = "file:./" + TestNcMLRead.topDir + "aggExistingWcoords.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" testNcmlDatasetWcoords.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
  logger.debug(" testNcmlDatasetWcoords.closed ");
}
 

private void readNext() throws IOException {
  int left = Math.min(recnum, readStart + readAtaTime); // dont go over recnum
  int need = left - readStart; // how many to read this time
  try {
    as = readStructure(readStart, need);
    if (NetcdfFile.debugStructureIterator)
      System.out.println("readNext " + count + " " + readStart);

  } catch (InvalidRangeException e) {
    log.error("Structure.IteratorRank1.readNext() ", e);
    throw new IllegalStateException("Structure.Iterator.readNext() ", e);
  } // cant happen
  readStart += need;
  readCount = 0;
}
 

static void openFile(String filename, Average avg, boolean enhance) throws IOException, InvalidRangeException {
  try {
    long start = System.nanoTime();
    NetcdfFile ncfile = enhance ? NetcdfDataset.openDataset(filename) : NetcdfDataset.openFile(filename, null);
    long end = System.nanoTime();
    double took = (double) ((end - start)) / 1000 / 1000 / 1000;
    ncfile.close();
    if (avg != null)
      avg.add(took);
  } catch (Exception e) {
    System.out.println("BAD " + filename);
    e.printStackTrace();
  }
}
 

/**
 * Test reading a section of data from an unsigned variable
 * 
 * @throws IOException
 * @throws InvalidRangeException
 */
@Test
public void testIsUnsignedReadSection() throws IOException, InvalidRangeException {
  // this worked as of 4.6.7, so no bug here...
  // assert v.isUnsigned();

  int[] shape = new int[] {1, 10, 20};
  Section section = new Section(shape);
  Array data = v.read(section);
  // this is the failure for https://github.com/Unidata/thredds/issues/695
  assert data.isUnsigned();
}
 

@Test
public void gcsPartialReadAquireFile() throws IOException, InvalidRangeException {
  DatasetUrl durl = DatasetUrl.findDatasetUrl(GCS_G16_S3_URI);
  try (NetcdfFile ncf = NetcdfDatasets.acquireFile(durl, null)) {
    testPartialReadGoes16S3(ncf);
  }
}