Java Code Examples for ucar.ma2.InvalidRangeException

Array readArray() throws IOException {
  int[] shape = mds.dimLength;
  DataType dataType = typeInfo.dataType;
  Layout layout;
  try {
    if (isChunked) {
      layout = new H5tiledLayout(this, dataType, new Section(shape));
    } else {
      layout = new LayoutRegular(dataPos, dataType.getSize(), shape, null);
    }
  } catch (InvalidRangeException e) {
    // cant happen because we use null for wantSection
    throw new IllegalStateException();
  }
  Object data = IospHelper.readDataFill(raf, layout, dataType, getFillValue(), typeInfo.endian, false);
  return Array.factory(dataType, shape, data);
}
 

@Test
public void testUpdateLastModified() throws IOException, InvalidRangeException, InterruptedException {
  // make sure that the extra file is not in the agg
  move(extraFile);

  // open the agg
  NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null);
  long start = ncfile.getLastModified();

  // now make sure that the extra file is in the agg
  moveBack(extraFile);

  // reread
  long end = ncfile.getLastModified();
  assert (end > start);

  // again
  long end2 = ncfile.getLastModified();
  assert (end == end2);

  ncfile.close();
}
 

/**
 * Read a section of the local Variable.
 *
 * @param mainv aggregated Variable
 * @param cancelTask let user cancel
 * @param section reletive to the local Variable
 * @return the complete Array for mainv
 * @throws IOException on I/O error
 * @throws InvalidRangeException on section error
 */
protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section)
    throws IOException, InvalidRangeException {
  NetcdfFile ncd = null;
  try {
    ncd = acquireFile(cancelTask);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;

    Variable v = findVariable(ncd, mainv);
    if (debugRead) {
      Section want = new Section(section);
      System.out.printf("Agg.read(%s) %s from %s in %s%n", want, mainv.getNameAndDimensions(),
          v.getNameAndDimensions(), getLocation());
    }

    return v.read(section);

  } finally {
    close(ncd);
  }
}
 

@Category(NeedsCdmUnitTest.class)
public void testGroupAndMembers() throws InvalidRangeException, IOException {
  NetcdfFile ncfile = NetcdfFiles.open(TestDir.cdmUnitTestDir + "formats/netcdf4/compound/simple_nc4.nc4");
  Variable v = ncfile.findVariable("grp1/data");
  assert v != null;

  ParsedSectionSpec spec = ParsedSectionSpec.parseVariableSection(ncfile, "grp1/data");
  System.out.printf("%s%n", spec);
  assert spec.section.equals(v.getShapeAsSection());

  spec = ParsedSectionSpec.parseVariableSection(ncfile, "grp2/data.i1");
  System.out.printf("%s%n", spec);

  Variable s = ncfile.findVariable("grp2/data");
  assert spec.section.equals(s.getShapeAsSection());

  v = ncfile.findVariable("grp2/data.i1");
  assert spec.child.section.equals(v.getShapeAsSection());

  ncfile.close();
}
 

@Test
public void testReadSubsection() throws IOException, InvalidRangeException {
  String location = TestDir.cdmUnitTestDir + "formats/netcdf4/ncom_relo_fukushima_1km_tmp_2011040800_t000.nc4";
  try (NetcdfFile ncfile = NetcdfFiles.open(location); NetcdfFile jni = openJni(location)) {

    jni.setLocation(location + " (jni)");

    // float salinity(time=1, depth=40, lat=667, lon=622);
    Array data1 = read(ncfile, "salinity", "0,11:12,22,:");
    // NCdumpW.printArray(data1);
    System.out.printf("Read from jni%n");
    Array data2 = read(jni, "salinity", "0,11:12,22,:");
    assert MAMath.nearlyEquals(data1, data2);
    System.out.printf("data is equal%n");
  }
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInFilenameOrNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

private StructureData nextStructureData() {
  StructureDataW sdata = new StructureDataW(members);

  for (Variable var : vars) {
    Section.Builder sb = Section.builder();
    try {
      sb.appendRange(outerIndex, outerIndex);
      sb.appendRange(count, count);
      for (int i = 2; i < var.getRank(); i++)
        sb.appendRangeAll();
      Array data = var.read(sb.build());
      sdata.setMemberData(var.getShortName(), data);

    } catch (InvalidRangeException | IOException e) {
      throw new RuntimeException(e);
    }

  }

  return sdata;
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "file:./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDataset.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

@Test
public void test1() throws IOException, InvalidRangeException {
  String filename = "file:./" + TestNcMLRead.topDir + "aggSynthetic.xml";
  NetcdfFile ncfile = NcMLReaderNew.readNcML(filename, null, null).build();

  Variable v = ncfile.findVariable("time");
  assert v != null;
  String testAtt = v.findAttributeString("units", null);
  assert testAtt != null;
  assert testAtt.equals("months since 2000-6-16 6:00");

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile, "T");
  testReadSlice(ncfile, "T");

  ncfile.close();
}
 

public void testReadSlice(NetcdfFile ncfile, int[] origin, int[] shape) throws IOException, InvalidRangeException {

    Variable v = ncfile.findVariable("T");

    Array data = v.read(origin, shape);
    assert data.getRank() == 3;
    assert data.getSize() == shape[0] * shape[1] * shape[2];
    assert data.getShape()[0] == shape[0] : data.getShape()[0] + " " + shape[0];
    assert data.getShape()[1] == shape[1];
    assert data.getShape()[2] == shape[2];
    assert data.getElementType() == double.class;

    Index tIndex = data.getIndex();
    for (int i = 0; i < shape[0]; i++)
      for (int j = 0; j < shape[1]; j++)
        for (int k = 0; k < shape[2]; k++) {
          double val = data.getDouble(tIndex.set(i, j, k));
          Assert2.assertNearlyEquals(val, 100 * (i + origin[0]) + 10 * j + k);
        }

  }
 

@Test
public void testRewriteHeader3() throws IOException, InvalidRangeException {
  String filename = tempFolder.newFile().getAbsolutePath();

  try (NetcdfFileWriter file = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, filename)) {
    file.addGlobalAttribute("att8", "1234567890");
    file.setExtraHeaderBytes(10);
    file.create();

    file.setRedefineMode(true);
    file.addGlobalAttribute("att8", "123456789012345");
    boolean rewriteAll = file.setRedefineMode(false);
    assert !rewriteAll;

    Attribute att = file.findGlobalAttribute("att8");
    assert att != null;
    assert att.getStringValue().equals("123456789012345") : att.getStringValue();
  }
}
 

static void testAllInDir(File dir, MClosure closure) throws IOException, InvalidRangeException {
  File[] fa = dir.listFiles();
  if (fa == null || fa.length == 0)
    return;

  List<File> list = Arrays.asList(fa);
  Collections.sort(list);

  for (File f : list) {
    if (f.isDirectory())
      testAllInDir(f, closure);
    else {
      closure.run(f.getPath());
    }
  }
}
 

static int timeDataRead(String remote, int stride) throws IOException, InvalidRangeException {
  logger.debug("--CdmRemote Read {} stride={}", remote, stride);

  try (NetcdfFile ncremote = new CdmRemote(remote)) {

    String gridName = "10_metre_V_wind_component_surface";
    Variable vs = ncremote.findVariable(gridName);
    Assert.assertNotNull(gridName, vs);

    Section section = vs.getShapeAsSection();
    Assert.assertEquals(3, section.getRank());
    Section want = Section.builder().appendRange(1).appendRange(section.getRange(1).copyWithStride(stride))
        .appendRange(section.getRange(2).copyWithStride(stride)).build();

    long start = System.currentTimeMillis();

    Array data = vs.read(want);

    long took = System.currentTimeMillis() - start;
    logger.debug("took={} size={}", took, data.getSize());

  }
  return 1;
}
 

/**
 * Write String data to a CHAR variable.
 *
 * @param v variable to write to
 * @param origin offset to start writing, ignore the strlen dimension.
 * @param values write this array; must be ArrayObject of String
 * @throws IOException if I/O error
 * @throws InvalidRangeException if values Array has illegal shape
 */
public void writeStringDataToChar(Variable v, int[] origin, Array values) throws IOException, InvalidRangeException {
  if (values.getElementType() != String.class)
    throw new IllegalArgumentException("values must be an ArrayObject of String ");

  if (v.getDataType() != DataType.CHAR)
    throw new IllegalArgumentException("variable " + v.getFullName() + " is not type CHAR");

  int rank = v.getRank();
  int strlen = v.getShape(rank - 1);

  // turn it into an ArrayChar
  ArrayChar cvalues = ArrayChar.makeFromStringArray((ArrayObject) values, strlen);

  int[] corigin = new int[rank];
  System.arraycopy(origin, 0, corigin, 0, rank - 1);

  write(v, corigin, cvalues);
}
 

public static List<ucar.ma2.Range> createCDMRanges(List<Slice> slices) throws DapException {
  List<ucar.ma2.Range> cdmranges = new ArrayList<Range>();
  for (int i = 0; i < slices.size(); i++) {
    Slice r = slices.get(i);
    try {
      ucar.ma2.Range cmdr;
      cmdr = new ucar.ma2.Range((int) r.getFirst(), (int) r.getLast(), (int) r.getStride());
      cdmranges.add(cmdr);
    } catch (InvalidRangeException ire) {
      throw new DapException(ire);
    }
  }
  return cdmranges;
}
 

static void doAll(String url, Stat tstat1, Stat tstat2) throws IOException, InvalidRangeException {
  // testCdmremoteCompress(url, stat1.setName("compress"), stat2.setName("nocomprs"), 10, true);
  System.out.printf("%n------ filename %s%n", url);

  Stat stat1 = new Stat(tstat1.getName(), false);
  Stat stat2 = new Stat(tstat2.getName(), false);

  compare(url, stat1, stat2, 5, true);

  System.out.printf(" %s MB/sec%n", stat1);
  System.out.printf(" %s MB/sec%n", stat2);

  tstat1.add(stat1);
  tstat2.add(stat2);
}
 

@Test
public void testNcmlDataset() throws IOException, InvalidRangeException {
  String filename = "file:./" + TestNcMLRead.topDir + "aggExisting.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

private void writeData(BeanTable from, File name) {
  Variable v = getCurrentVariable(from);
  if (v == null || name == null)
    return;

  try (FileOutputStream stream = new FileOutputStream(name)) {
    WritableByteChannel channel = stream.getChannel();
    v.readToByteChannel(v.getShapeAsSection(), channel);
    System.out.printf("Write ok to %s%n", name);

  } catch (InvalidRangeException | IOException e) {
    e.printStackTrace();
  }

}
 

/**
 * Test reading a section of data from an unsigned variable
 * 
 * @throws IOException
 * @throws InvalidRangeException
 */
@Test
public void testIsUnsignedReadSection() throws IOException, InvalidRangeException {
  // this worked as of 4.6.7, so no bug here...
  // assert v.isUnsigned();

  int[] shape = new int[] {1, 10, 20};
  Section section = new Section(shape);
  Array data = v.read(section);
  // this is the failure for https://github.com/Unidata/thredds/issues/695
  assert data.isUnsigned();
}
 

public long makeFile(String location, ucar.nc2.dt.GridDataset gds, List<String> gridList, LatLonRect llbb,
    int horizStride, Range zRange, CalendarDateRange dateRange, int stride_time, boolean addLatLon)
    throws IOException, InvalidRangeException {

  return makeOrTestSize(location, gds, gridList, llbb, horizStride, zRange, dateRange, stride_time, addLatLon, false,
      NetcdfFileWriter.Version.netcdf3);
}
 

@Test
public void testNcmlDatasetNoProtocolInFilename() throws IOException, InvalidRangeException {
  String filename = "./" + TestNcMLRead.topDir + "aggExisting.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

@Test
public void checkPaddingOnWriteReadOneDimCharArrayOneOfTwoRecordVars() throws IOException, InvalidRangeException {
  File tmpDataDir = tempFolder.newFolder();
  File testFile = new File(tmpDataDir, "file.nc");

  NetcdfFormatWriter.Builder writerb = NetcdfFormatWriter.createNewNetcdf3(testFile.getPath());
  writerb.addDimension(Dimension.builder().setName("v").setIsUnlimited(true).build());
  writerb.addDimension(Dimension.builder("s", 3).build());

  Variable.Builder v = writerb.addVariable("v", DataType.CHAR, "v s");
  assertEquals(1, v.getElementSize());
  writerb.addVariable("v2", DataType.CHAR, "v");

  char[] data = {1, 2, 3, 40, 41, 42, 50, 51, 52, 60, 61, 62};

  try (NetcdfFormatWriter ncfWriteable = writerb.build()) {
    Variable var = ncfWriteable.findVariable("v");

    Array dataArray = Array.factory(DataType.CHAR, new int[] {4, 3}, data);
    ncfWriteable.write(var.getFullNameEscaped(), dataArray);
  }

  try (NetcdfFile ncf = NetcdfFiles.open(testFile.getPath())) {
    Variable readVar = ncf.findVariable("v");
    assertEquals(readVar.getDataType(), DataType.CHAR);
    assertEquals(1, readVar.getElementSize());
    Variable readVar2 = ncf.findVariable("v2");
    assertEquals(readVar2.getDataType(), DataType.CHAR);
    assertEquals(1, readVar2.getElementSize());

    int[] org = {0, 0};
    char[] readdata = (char[]) readVar.read(org, readVar.getShape()).copyTo1DJavaArray();

    assertArrayEquals(data, readdata);
  }
}
 

@Test
public void testAggExisting() throws IOException, InvalidRangeException {
  String endpoint = TestOnLocalServer.withHttpPath("dodsC/ExampleNcML/Agg.nc");
  logger.debug("{}", endpoint);

  try (NetcdfFile ncfile = NetcdfDatasets.openFile(endpoint, null)) {
    Variable v = ncfile.findVariable("time");
    assert v != null;
    assert v.getDataType() == DataType.DOUBLE;

    String units = v.getUnitsString();
    assert units != null;
    assert units.equals("Hour since 2006-09-25T06:00:00Z");

    int count = 0;
    Array data = v.read();
    logger.debug(Ncdump.printArray(data, "time", null));

    while (data.hasNext()) {
      Assert2.assertNearlyEquals(data.nextInt(), (count + 1) * 3);
      count++;
    }

    // test attributes added in NcML
    String testAtt = ncfile.getRootGroup().findAttributeString("ncmlAdded", null);
    assert testAtt != null;
    assert testAtt.equals("stuff");

    v = ncfile.findVariable("lat");
    assert v != null;
    testAtt = v.findAttributeString("ncmlAdded", null);
    assert testAtt != null;
    assert testAtt.equals("lat_stuff");
  }
}
 

@Test
public void testBestStride() throws IOException, InvalidRangeException {
  logger.debug("testBestStride {}", endpoint);

  try (FeatureDatasetCoverage cc = CoverageDatasetFactory.open(endpoint)) {
    Assert.assertNotNull(endpoint, cc);
    CoverageCollection gcs = cc.findCoverageDataset(FeatureType.GRID);
    Assert.assertNotNull("gcs", gcs);
    Coverage cover = gcs.findCoverage(covName);
    Assert.assertNotNull(covName, cover);
    CoverageCoordSys csys = cover.getCoordSys();
    int[] csysShape = csys.getShape();
    logger.debug("csys shape = {}", Arrays.toString(csysShape));

    SubsetParams params = new SubsetParams().setHorizStride(2);
    Optional<CoverageCoordSys> opt = csys.subset(params);
    if (!opt.isPresent()) {
      logger.debug("err={}", opt.getErrorMessage());
      assert false;
    }

    CoverageCoordSys subsetCoordSys = opt.get();
    int[] subsetShape = subsetCoordSys.getShape();
    logger.debug("csysSubset shape = {}", Arrays.toString(subsetShape));

    int n = csysShape.length;
    csysShape[n - 1] = (csysShape[n - 1] + 1) / 2;
    csysShape[n - 2] = (csysShape[n - 2] + 1) / 2;

    Assert.assertArrayEquals(csysShape, subsetShape);
  }
}
 

private static void writeValues(NetcdfFile netcdfToAdd, NetcdfFileWriter netcdfWriter) throws IOException, InvalidRangeException {
	List<Variable> variables = netcdfToAdd.getVariables();
	for (Variable variable : variables) {
		String fullNameEscaped = variable.getFullNameEscaped();
		Variable netcdfFileVariable = netcdfWriter.findVariable(fullNameEscaped);
		Array read = variable.read();
		netcdfWriter.write(netcdfFileVariable, read);
	}
}
 

@Test
public void awsPartialReadDataset() throws IOException, InvalidRangeException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfDataset ncd = NetcdfDatasets.openDataset(AWS_G16_S3_URI_FULL)) {
    testPartialReadGoes16S3(ncd);
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
public void awsPartialReadFileSimple() throws IOException, InvalidRangeException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_SIMPLE)) {
    testPartialReadGoes16S3(ncfile);
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
public void checkPaddingOnWriteReadOneDimByteArrayOneOfTwoRecordVars() throws IOException, InvalidRangeException {
  File tmpDataDir = tempFolder.newFolder();
  File testFile = new File(tmpDataDir, "file.nc");

  NetcdfFormatWriter.Builder writerb = NetcdfFormatWriter.createNewNetcdf3(testFile.getPath());
  writerb.addDimension(Dimension.builder().setName("v").setIsUnlimited(true).build());
  Variable.Builder v = writerb.addVariable("v", DataType.BYTE, "v");
  assertEquals(1, v.getElementSize());
  writerb.addVariable("v2", DataType.BYTE, "v");

  byte[] data = {1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -2, -3, -4, -5, -6, -7, -8, -9};

  try (NetcdfFormatWriter ncfWriteable = writerb.build()) {
    Variable var = ncfWriteable.findVariable("v");
    assert var != null;

    Variable var2 = ncfWriteable.findVariable("v2");
    assert var2 != null;

    Array dataArray = Array.factory(DataType.BYTE, new int[] {data.length}, data);
    ncfWriteable.write(var.getFullNameEscaped(), dataArray);
  }

  try (NetcdfFile ncf = NetcdfFiles.open(testFile.getPath())) {
    Variable readVar = ncf.findVariable("v");
    assertEquals(readVar.getDataType(), DataType.BYTE);
    assertEquals(1, readVar.getElementSize());

    Variable readVar2 = ncf.findVariable("v2");
    assertEquals(readVar2.getDataType(), DataType.BYTE);
    assertEquals(1, readVar2.getElementSize());

    int[] org = {0};
    byte[] readdata = (byte[]) readVar.read(org, readVar.getShape()).copyTo1DJavaArray();

    assertArrayEquals(data, readdata);
  }
}
 

private void fillNonRecordVariables() throws IOException {
  // run through each variable
  for (Variable v : ncfile.getVariables()) {
    if (v.isUnlimited())
      continue;
    try {
      writeData(v, v.getShapeAsSection(), makeConstantArray(v));
    } catch (InvalidRangeException e) {
      e.printStackTrace(); // shouldnt happen
    }
  }
}
 

@Test
public void gcsPartialReadAquireFile() throws IOException, InvalidRangeException {
  DatasetUrl durl = DatasetUrl.findDatasetUrl(GCS_G16_S3_URI);
  try (NetcdfFile ncf = NetcdfDatasets.acquireFile(durl, null)) {
    testPartialReadGoes16S3(ncf);
  }
}