ucar.ma2.Section Java Examples

@Test
public void testWithDims() {
  try {
    // Must set dimension first
    VariableDS.builder().setName("name").setDataType(DataType.FLOAT).setDimensionsByName("dim1 dim2")
        .build(makeDummyGroup());
    fail();
  } catch (Exception e) {
    // ok
  }

  Group group =
      Group.builder().addDimension(Dimension.builder().setName("dim1").setLength(7).setIsUnlimited(true).build())
          .addDimension(Dimension.builder().setName("dim2").setLength(27).build()).build();
  List<Dimension> varDims = group.makeDimensionsList("dim1 dim2");

  VariableDS var =
      VariableDS.builder().setName("name").setDataType(DataType.FLOAT).addDimensions(varDims).build(group);
  assertThat(var.getDataType()).isEqualTo(DataType.FLOAT);
  assertThat(var.getShortName()).isEqualTo("name");
  assertThat(var.isScalar()).isFalse();
  assertThat(var.isUnlimited()).isTrue();
  assertThat(var.getShape()).isEqualTo(new int[] {7, 27});
  assertThat(var.getShapeAll()).isEqualTo(new int[] {7, 27});
  assertThat(var.getShapeAsSection()).isEqualTo(new Section(new int[] {7, 27}));
}
 

public static int readAllData(NetcdfFile ncfile) throws IOException {
  logger.debug("------Reading ncfile {}", ncfile.getLocation());
  try {
    for (Variable v : ncfile.getVariables()) {
      if (v.getSize() > max_size) {
        Section s = makeSubset(v);
        logger.debug("  Try to read variable {} size={} section={}", v.getNameAndDimensions(), v.getSize(), s);
        v.read(s);
      } else {
        logger.debug("  Try to read variable {} size={}", v.getNameAndDimensions(), v.getSize());
        v.read();
      }
    }

    return 1;
  } catch (InvalidRangeException e) {
    throw new RuntimeException(e);
  }
}
 

private void checkConformance(Coverage gridSubset, GeoReferencedArray geo, String where) {
  CoverageCoordSys csys = gridSubset.getCoordSys();

  CoverageCoordSys csysData = geo.getCoordSysForData();

  Section s = new Section(csys.getShape());
  Section so = new Section(csysData.getShape());

  boolean ok = s.conformal(so);

  int[] dataShape = geo.getData().getShape();
  Section sdata = new Section(dataShape);
  boolean ok2 = s.conformal(sdata);

  if (!ok || !ok2)
    logger.warn("CFGridCoverageWriter checkConformance fails " + where);
}
 

/**
 * Read the data for the variable
 *
 * @param v2 Variable to read
 * @param section section infomation
 * @return Array of data
 * @throws IOException problem reading from file
 * @throws InvalidRangeException invalid Range
 */
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {

  Array dataArray = Array.factory(DataType.FLOAT, section.getShape());
  GradsVariable gradsVar = findVar(v2);
  if (gradsVar == null)
    throw new IOException();

  // Canonical ordering is ens, time, level, lat, lon
  int rangeIdx = 0;
  Range ensRange = (gradsDDF.getEnsembleDimension() != null) ? section.getRange(rangeIdx++) : new Range(0, 0);
  Range timeRange = (section.getRank() > 2) ? section.getRange(rangeIdx++) : new Range(0, 0);
  Range levRange = (gradsVar.getNumLevels() > 0) ? section.getRange(rangeIdx++) : new Range(0, 0);
  Range yRange = section.getRange(rangeIdx++);
  Range xRange = section.getRange(rangeIdx);

  IndexIterator ii = dataArray.getIndexIterator();

  for (int ensIdx : ensRange)
    for (int timeIdx : timeRange)
      for (int levelIdx : levRange)
        readXY(v2, ensIdx, timeIdx, levelIdx, yRange, xRange, ii);

  return dataArray;
}
 

/**
 * Reads a subsampled sub-area of the variable.
 * Array elements are in inverse of netCDF order.
 *
 * @param  area         indices of cell values to read along each dimension, in "natural" order.
 * @param  subsampling  subsampling along each dimension. 1 means no subsampling.
 * @return the data as an array of a Java primitive type.
 */
@Override
public Vector read(final GridExtent area, final int[] subsampling) throws IOException, DataStoreException {
    int n = area.getDimension();
    final int[] lower = new int[n];
    final int[] size  = new int[n];
    final int[] sub   = new int[n--];
    for (int i=0; i<=n; i++) {
        final int j = (n - i);
        lower[j] = Math.toIntExact(area.getLow(i));
        size [j] = Math.toIntExact(area.getSize(i));
        sub  [j] = subsampling[i];
    }
    final Array array;
    try {
        array = variable.read(new Section(lower, size, sub));
    } catch (InvalidRangeException e) {
        throw new DataStoreException(e);
    }
    return Vector.create(get1DJavaArray(array), variable.isUnsigned());
}
 

private Array readVlenData(Variable v, Section section, DataStorage dataStorage) throws IOException {
  raf.seek(dataStorage.filePos);
  int nelems = readVInt(raf);
  Array[] result = new Array[nelems];

  for (int elem = 0; elem < nelems; elem++) {
    int dsize = readVInt(raf);
    byte[] data = new byte[dsize];
    raf.readFully(data);
    Array dataArray = Array.factory(v.getDataType(), (int[]) null, ByteBuffer.wrap(data));
    result[elem] = dataArray;
  }
  // return Array.makeObjectArray(v.getDataType(), result[0].getClass(), new int[]{nelems}, result);
  return Array.makeVlenArray(new int[] {nelems}, result);
  // return dataArray.section(section.getRanges());
}
 

static public int readAllData(NetcdfFile ncfile) throws IOException {
  logger.debug("------Reading ncfile {}", ncfile.getLocation());
  try {
    for (Variable v : ncfile.getVariables()) {
      if (v.getSize() > max_size) {
        Section s = makeSubset(v);
        logger.debug("  Try to read variable {} size={} section={}", v.getNameAndDimensions(), v.getSize(), s);
        v.read(s);
      } else {
        logger.debug("  Try to read variable {} size={}", v.getNameAndDimensions(), v.getSize());
        v.read();
      }
    }

    return 1;
  } catch (InvalidRangeException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Use this when this is a one dimensional array of Structures, or you are doing the index calculation yourself for
 * a multidimension array. This will read only the ith structure, and return the data as a StructureData object.
 * 
 * @param index index into 1D array
 * @return ith StructureData
 * @throws java.io.IOException on read error
 * @throws ucar.ma2.InvalidRangeException if index out of range
 */
public StructureData readStructure(int index) throws IOException, ucar.ma2.InvalidRangeException {
  Section.Builder sb = Section.builder();

  if (getRank() == 1) {
    sb.appendRange(index, index);

  } else if (getRank() > 1) {
    Index ii = Index.factory(shape); // convert to nD index
    ii.setCurrentCounter(index);
    int[] origin = ii.getCurrentCounter();
    for (int anOrigin : origin)
      sb.appendRange(anOrigin, anOrigin);
  }

  Array dataArray = read(sb.build());
  ArrayStructure data = (ArrayStructure) dataArray;
  return data.getStructureData(0);
}
 

/**
 * Read a section of the local Variable.
 *
 * @param mainv aggregated Variable
 * @param cancelTask let user cancel
 * @param section reletive to the local Variable
 * @return the complete Array for mainv
 * @throws IOException on I/O error
 * @throws InvalidRangeException on section error
 */
protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section)
    throws IOException, InvalidRangeException {
  NetcdfFile ncd = null;
  try {
    ncd = acquireFile(cancelTask);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;

    Variable v = findVariable(ncd, mainv);
    if (debugRead) {
      Section want = new Section(section);
      System.out.printf("Agg.read(%s) %s from %s in %s%n", want, mainv.getNameAndDimensions(),
          v.getNameAndDimensions(), getLocation());
    }

    return v.read(section);

  } finally {
    close(ncd);
  }
}
 

Array readArray() throws IOException {
  int[] shape = mds.dimLength;
  DataType dataType = typeInfo.dataType;
  Layout layout;
  try {
    if (isChunked) {
      layout = new H5tiledLayout(this, dataType, new Section(shape));
    } else {
      layout = new LayoutRegular(dataPos, dataType.getSize(), shape, null);
    }
  } catch (InvalidRangeException e) {
    // cant happen because we use null for wantSection
    throw new IllegalStateException();
  }
  Object data = IospHelper.readDataFill(raf, layout, dataType, getFillValue(), typeInfo.endian, false);
  return Array.factory(dataType, shape, data);
}
 

private void fillRecordVariables(int recStart, int recEnd) throws IOException, InvalidRangeException {
  // do each record completely, should be a bit more efficient

  for (int i = recStart; i < recEnd; i++) { // do one record at a time
    Range r = new Range(i, i);

    // run through each variable
    for (Variable v : ncfile.getVariables()) {
      if (!v.isUnlimited() || (v instanceof Structure))
        continue;
      Section.Builder recordSection = Section.builder().appendRanges(v.getRanges());
      recordSection.setRange(0, r);
      writeData(v, recordSection.build(), makeConstantArray(v));
    }
  }
}
 

/**
 * Do not call this directly, use Variable.read() !!
 * Ranges must be filled (no nulls)
 */
protected Array readData(Variable v, Section ranges) throws IOException, InvalidRangeException {
  long start = 0;
  if (showRequest) {
    log.info("Data request for variable: {} section {}...", v.getFullName(), ranges);
    start = System.currentTimeMillis();
  }

  /*
   * if (unlocked) {
   * String info = cache.getInfo(this);
   * throw new IllegalStateException("File is unlocked - cannot use\n" + info);
   * }
   */

  if (iosp == null) {
    throw new IOException("iosp is null, perhaps file has been closed. Trying to read variable " + v.getFullName());
  }
  Array result = iosp.readData(v, ranges);

  if (showRequest) {
    long took = System.currentTimeMillis() - start;
    log.info(" ...took= {} msecs", took);
  }
  return result;
}
 

private Array read(TimeInventory.Instance timeInstance, String fullNameEsc, List<Range> innerSection,
    HashMap<String, NetcdfDataset> openFilesRead) throws IOException, InvalidRangeException {
  NetcdfFile ncfile = open(timeInstance.getDatasetLocation(), openFilesRead);
  if (ncfile == null)
    return null; // file might be deleted ??

  Variable v = ncfile.findVariable(fullNameEsc);
  if (v == null)
    return null; // v could be missing, return missing data i think

  // assume time is first dimension LOOK: out of-order; ensemble; section different ??
  Range timeRange = new Range(timeInstance.getDatasetIndex(), timeInstance.getDatasetIndex());
  Section.Builder sb = Section.builder().appendRanges(innerSection);
  sb.insertRange(0, timeRange);
  return v.read(sb.build());
}
 

private StructureData nextStructureData() {
  StructureDataW sdata = new StructureDataW(members);

  for (Variable var : vars) {
    Section.Builder sb = Section.builder();
    try {
      sb.appendRange(outerIndex, outerIndex);
      sb.appendRange(count, count);
      for (int i = 2; i < var.getRank(); i++)
        sb.appendRangeAll();
      Array data = var.read(sb.build());
      sdata.setMemberData(var.getShortName(), data);

    } catch (InvalidRangeException | IOException e) {
      throw new RuntimeException(e);
    }

  }

  return sdata;
}
 

/**
 * Constructor.
 *
 * @param segPos starting address of each segment.
 * @param segSize number of bytes in each segment. Assume multiple of elemSize
 * @param elemSize size of an element in bytes.
 * @param srcShape shape of the entire data array.
 * @param wantSection the wanted section of data
 * @throws ucar.ma2.InvalidRangeException if ranges are misformed
 */
public LayoutSegmented(long[] segPos, int[] segSize, int elemSize, int[] srcShape, Section wantSection)
    throws InvalidRangeException {
  assert segPos.length == segSize.length;
  this.segPos = segPos;

  int nsegs = segPos.length;
  segMin = new long[nsegs];
  segMax = new long[nsegs];
  long totalElems = 0;
  for (int i = 0; i < nsegs; i++) {
    assert segPos[i] >= 0;
    assert segSize[i] > 0;
    assert (segSize[i] % elemSize) == 0;

    segMin[i] = totalElems;
    totalElems += segSize[i];
    segMax[i] = totalElems;
  }
  assert totalElems >= Index.computeSize(srcShape) * elemSize;

  chunker = new IndexChunker(srcShape, wantSection);
  this.total = chunker.getTotalNelems();
  this.done = 0;
  this.elemSize = elemSize;
}
 

@Test
public void testWithDims() {
  try {
    // Must set dimension first
    Variable.builder().setName("name").setDataType(DataType.FLOAT).setDimensionsByName("dim1 dim2")
        .build(makeDummyGroup());
    fail();
  } catch (Exception e) {
    // ok
  }

  Group.Builder gb = Group.builder().addDimension(Dimension.builder("dim1", 7).setIsUnlimited(true).build())
      .addDimension(new Dimension("dim2", 27));

  Variable var = Variable.builder().setName("name").setDataType(DataType.FLOAT).setParentGroupBuilder(gb)
      .setDimensionsByName("dim1 dim2").build(gb.build());
  assertThat(var.getDataType()).isEqualTo(DataType.FLOAT);
  assertThat(var.getShortName()).isEqualTo("name");
  assertThat(var.isScalar()).isFalse();
  assertThat(var.isUnlimited()).isTrue();
  assertThat(var.getShape()).isEqualTo(new int[] {7, 27});
  assertThat(var.getShapeAsSection()).isEqualTo(new Section(new int[] {7, 27}));
}
 

@Override
public Array readData(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException {
  if (v2 instanceof Structure)
    return readRecordData((Structure) v2, section);

  Vinfo vinfo = (Vinfo) v2.getSPobject();
  DataType dataType = v2.getDataType();

  Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
      : new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);

  if (layout.getTotalNelems() == 0) {
    return Array.factory(dataType, section.getShape());
  }

  Object data = readData(layout, dataType);
  return Array.factory(dataType, section.getShape(), data);
}
 

private static void subsetVariable(Variable v, Section s, Array fullData) throws IOException, InvalidRangeException {
  System.out.println("   section=" + s);

  // read just that
  Array sdata = v.read(s);
  assert sdata.getRank() == s.getRank();
  int[] sshape = sdata.getShape();
  for (int i = 0; i < sshape.length; i++)
    assert sshape[i] == s.getShape(i);

  // compare with logical section
  Array Asection = fullData.sectionNoReduce(s.getRanges());
  int[] ashape = Asection.getShape();
  assert (ashape.length == sdata.getRank());
  for (int i = 0; i < ashape.length; i++)
    assert sshape[i] == ashape[i];

  CompareNetcdf2.compareData(v.getShortName(), sdata, Asection);
}
 

long computeArraySize(SDArray da) throws Exception {
  assert (da.getContainerVar() instanceof DPrimitive);
  BaseType base = da.getPrimitiveVector().getTemplate();
  DataType dtype = DODSNetcdfFile.convertToNCType(base, false);
  int elemSize = dtype.getSize();
  int n = da.numDimensions();
  List<Range> ranges = new ArrayList<>(n);
  long size = 0;
  for (int i = 0; i < n; i++) {
    ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i)));
  }
  Section s = new Section(ranges);
  size += s.computeSize() * elemSize;

  return size;
}
 

/** Make a ucar.ma2.Section.Builder from an ordered set of Dimension objects. */
public static ucar.ma2.Section.Builder makeSectionFromDimensions(Iterable<Dimension> dimensions) {
  try {
    Section.Builder builder = Section.builder();
    for (Dimension d : dimensions) {
      int len = d.getLength();
      if (len > 0)
        builder.appendRange(new Range(d.getShortName(), 0, len - 1));
      else if (len == 0)
        builder.appendRange(Range.EMPTY); // LOOK empty not named
      else {
        assert d.isVariableLength();
        builder.appendRange(Range.VLEN); // LOOK vlen not named
      }
    }
    return builder;

  } catch (InvalidRangeException e) {
    throw new IllegalStateException(e.getMessage());
  }
}
 

/**
 * Constructor.
 *
 * @param chunkIterator iterator over all available data chunks
 * @param chunkSize all chunks assumed to be the same size
 * @param elemSize size of an element in bytes.
 * @param wantSection the wanted section of data, contains a List of Range objects. Must be complete
 */
public LayoutTiled(DataChunkIterator chunkIterator, int[] chunkSize, int elemSize, Section wantSection) {
  this.chunkIterator = chunkIterator;
  this.chunkSize = chunkSize;
  this.elemSize = elemSize;
  this.want = wantSection;
  if (this.want.isVariableLength()) {
    // remove the varlen
    List<Range> newrange = new ArrayList<>(this.want.getRanges());
    newrange.remove(newrange.size() - 1);
    this.want = new Section(newrange);
  }

  this.totalNelems = this.want.computeSize();
  this.totalNelemsDone = 0;
}
 

/**
 * Constructor.
 *
 * @param startPos starting address of the entire data array.
 * @param elemSize size of an element in bytes.
 * @param recSize size of outer stride in bytes
 * @param srcShape shape of the entire data array. must have rank > 0
 * @param wantSection the wanted section of data
 * @throws ucar.ma2.InvalidRangeException if ranges are misformed
 */
public LayoutRegularSegmented(long startPos, int elemSize, long recSize, int[] srcShape, Section wantSection)
    throws InvalidRangeException {
  assert startPos > 0;
  assert elemSize > 0;
  assert recSize > 0;
  assert srcShape.length > 0;

  this.startPos = startPos;
  this.elemSize = elemSize;
  this.recSize = recSize;

  chunker = new IndexChunker(srcShape, wantSection);
  this.total = chunker.getTotalNelems();
  this.innerNelems = (srcShape[0] == 0) ? 0 : Index.computeSize(srcShape) / srcShape[0];
  this.done = 0;
}
 

/**
 * Read data from a top level Variable and send data to a WritableByteChannel.
 *
 * @param v2 Variable
 * @param section wanted section of data of Variable. The section list is a list
 *        of ucar.ma2.Range which define the requested data subset.
 * @param channel WritableByteChannel object - channel that can write bytes.
 * @return the number of bytes written, possibly zero.
 */
public long readToByteChannel11(ucar.nc2.Variable v2, Section section, WritableByteChannel channel)
    throws java.io.IOException {
  Array data = readData(v2, section);
  float[] ftdata = new float[(int) data.getSize()];
  byte[] bytedata = new byte[(int) data.getSize() * 4];
  IndexIterator iter = data.getIndexIterator();
  int i = 0;
  ByteBuffer buffer = ByteBuffer.allocateDirect(bytedata.length);
  while (iter.hasNext()) {
    ftdata[i] = iter.getFloatNext();
    bytedata[i] = new Float(ftdata[i]).byteValue();
    buffer.put(bytedata[i]);
    i++;
  }
  buffer = ByteBuffer.wrap(bytedata);
  // write the bytes to the channel
  int count = channel.write(buffer);
  // check if all bytes where written
  if (buffer.hasRemaining()) {
    // if not all bytes were written, move the unwritten bytes to the beginning and
    // set position just after the last unwritten byte
    buffer.compact();
  } else {
    buffer.clear();
  }
  return (long) count;
}
 

/**
 * Constructor.
 * This is for HDF5 chunked data storage. The data is read by chunk, for efficency.
 *
 * @param vinfo the vinfo object for this variable
 * @param dtype type of data. may be different from v2.
 * @param wantSection the wanted section of data, contains a List of Range objects, must be complete
 * @throws java.io.IOException on io error
 */
public H5tiledLayout(H5header.Vinfo vinfo, DataType dtype, Section wantSection) throws IOException {
  assert vinfo.isChunked;
  assert vinfo.btree != null;

  // we have to translate the want section into the same rank as the storageSize, in order to be able to call
  // Section.intersect(). It appears that storageSize (actually msl.chunkSize) may have an extra dimension, reletive
  // to the Variable.
  if ((dtype == DataType.CHAR) && (wantSection.getRank() < vinfo.storageSize.length)) {
    this.want = Section.builder().appendRanges(wantSection.getRanges()).appendRange(1).build();
  } else {
    this.want = wantSection;
  }

  // one less chunk dimension, except in the case of char
  int nChunkDims = (dtype == DataType.CHAR) ? vinfo.storageSize.length : vinfo.storageSize.length - 1;
  this.chunkSize = new int[nChunkDims];
  System.arraycopy(vinfo.storageSize, 0, chunkSize, 0, nChunkDims);

  this.elemSize = vinfo.storageSize[vinfo.storageSize.length - 1]; // last one is always the elements size
  if (debug)
    System.out.println(" H5tiledLayout: " + this);

  // create the data chunk iterator
  LayoutTiled.DataChunkIterator iter = vinfo.btree.getDataChunkIteratorNoFilter(this.want, nChunkDims);
  delegate = new LayoutTiled(iter, chunkSize, elemSize, this.want);
}
 

@Test
public void testChunkerTiled2() throws InvalidRangeException {
  Section dataSection = new Section("0:0, 40:59,  0:1353  ");
  Section wantSection = new Section("0:2, 22:3152,0:1350");
  IndexChunkerTiled index = new IndexChunkerTiled(dataSection, wantSection);
  while (index.hasNext()) {
    Layout.Chunk chunk = index.next();
    System.out.println(" " + chunk);
  }
}
 

public Array reallyRead(Variable client, Section section, CancelTask cancelTask)
    throws IOException, InvalidRangeException {
  try (NetcdfFile ncfile = open(location, null)) {
    Variable proxyV = findVariable(ncfile, client);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;
    return proxyV.read(section);
  }
}
 

protected long readToOutputStream(Variable v, Section section, OutputStream out)
    throws IOException, InvalidRangeException {

  // if (unlocked)
  // throw new IllegalStateException("File is unlocked - cannot use");

  if ((iosp == null) || v.hasCachedData())
    return IospHelper.copyToOutputStream(v.read(section), out);

  return iosp.readToOutputStream(v, section, out);
}
 

private long readRecordData(ucar.nc2.Structure s, Section section, WritableByteChannel out)
    throws java.io.IOException {
  long count = 0;

  /*
   * RegularIndexer index = new RegularIndexer( s.getShape(), recsize, recStart, section, recsize);
   * while (index.hasNext()) {
   * Indexer.Chunk chunk = index.next();
   * count += raf.readBytes( out, chunk.getFilePos(), chunk.getNelems() * s.getElementSize());
   * }
   */

  // LOOK this is the OTW layout based on netcdf-3
  // not sure this works but should give an idea of timing
  Range recordRange = section.getRange(0);
  /*
   * int stride = recordRange.stride();
   * if (stride == 1) {
   * int first = recordRange.first();
   * int n = recordRange.length();
   * if (false) System.out.println(" read record " + first+" "+ n * header.recsize+" bytes ");
   * return raf.readToByteChannel(out, header.recStart + first * header.recsize, n * header.recsize);
   *
   * } else {
   */

  for (int recnum : recordRange) {
    if (debugRecord)
      System.out.println(" read record " + recnum);
    raf.seek(header.recStart + recnum * header.recsize); // where the record starts
    count += raf.readToByteChannel(out, header.recStart + recnum * header.recsize, header.recsize);
  }
  // }

  return count;
}
 

@Override
public long readToByteChannel(ucar.nc2.Variable v2, Section section, WritableByteChannel channel)
    throws java.io.IOException, ucar.ma2.InvalidRangeException {

  if (v2 instanceof Structure)
    return readRecordData((Structure) v2, section, channel);

  Vinfo vinfo = (Vinfo) v2.getSPobject();
  DataType dataType = v2.getDataType();

  Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
      : new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);

  return readData(layout, dataType, channel);
}
 

@Override
public long readToOutputStream(ucar.nc2.Variable v2, Section section, OutputStream out)
    throws java.io.IOException, ucar.ma2.InvalidRangeException {

  Array data = readData(v2, section);
  return IospHelper.copyToOutputStream(data, out);
}