Java Code Examples for ucar.nc2.Variable#getSPobject()

public Array readData(Variable v2, Section section) throws IOException {
  Vgroup vgroup = (Vgroup) v2.getSPobject();

  Range scanRange = section.getRange(0);
  Range radialRange = section.getRange(1);
  Range gateRange = section.getRange(2);

  Array data = Array.factory(v2.getDataType(), section.getShape());
  IndexIterator ii = data.getIndexIterator();

  for (int scanIdx : scanRange) {
    Ray[] mapScan = vgroup.map[scanIdx];
    readOneScan(mapScan, radialRange, gateRange, vgroup.abbrev, ii);
  }

  return data;
}
 

private StructureData readStructure(Structure s, ArrayStructureW asw, long dataPos)
    throws IOException, InvalidRangeException {
  StructureDataW sdata = new StructureDataW(asw.getStructureMembers());
  if (debug)
    System.out.println(" readStructure " + s.getFullName() + " dataPos = " + dataPos);

  for (Variable v2 : s.getVariables()) {
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject();
    if (debug)
      System.out.println(" readStructureMember " + v2.getFullName() + " vinfo = " + vinfo);
    Array dataArray = readData(v2, dataPos + vinfo.dataPos, v2.getShapeAsSection());
    sdata.setMemberData(v2.getShortName(), dataArray);
  }

  return sdata;
}
 

public void showCompress(Formatter f) throws IOException {
  NetcdfFile ncfile = iosp.getNetcdfFile();

  Size totalSize = new Size(0, 0);
  for (Variable v : ncfile.getVariables()) {
    H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
    showCompress(v, vinfo, totalSize, f);
  }
  f.format("%n");
  f.format(" total bytes   = %d%n", totalSize.nominal);
  f.format(" total storage = %d%n", totalSize.storage);
  f.format(" compression = %f%n", totalSize.getRatio());
  f.format(" nchunks     = %d%n", totalSize.count);

  File raf = new File(ncfile.getLocation());
  f.format(" file size    = %d%n", raf.length());
  float overhead = totalSize.storage == 0 ? 0 : ((float) raf.length() / totalSize.storage);
  f.format(" overhead     = %f%n", overhead);
}
 

public void showCompress(Formatter f) throws IOException {
  Size totalSize = new Size(0, 0);
  for (Variable v : ncfile.getVariables()) {
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
    showCompress(v, vinfo, totalSize, f);
  }
  f.format("%n");
  f.format(" total bytes   = %d%n", totalSize.nominal);
  f.format(" total storage = %d%n", totalSize.storage);
  f.format(" compression = %f%n", totalSize.getRatio());
  f.format(" nchunks     = %d%n", totalSize.count);

  File raf = new File(ncfile.getLocation());
  f.format(" file size    = %d%n", raf.length());
  float overhead = totalSize.storage == 0 ? 0 : ((float) raf.length() / totalSize.storage);
  f.format(" overhead     = %f%n", overhead);
}
 

/**
 * Read the data for each variable passed in
 */
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {

  // subset
  Object data;
  Array outputData;
  byte[] vdata;
  NOWRadheader.Vinfo vinfo;
  ByteBuffer bos;
  List<Range> ranges = section.getRanges();

  vinfo = (NOWRadheader.Vinfo) v2.getSPobject();
  vdata = headerParser.getData((int) vinfo.hoff);

  bos = ByteBuffer.wrap(vdata);
  data = readOneScanData(bos, vinfo, v2.getShortName());
  outputData = Array.factory(v2.getDataType(), v2.getShape(), data);
  outputData = outputData.flip(1);

  // outputData = outputData.flip(2);
  return (outputData.sectionNoReduce(ranges).copy());

  // return outputData;
}
 

private void writeRecordData(ucar.nc2.Structure s, int recnum, StructureData sdata)
    throws java.io.IOException, ucar.ma2.InvalidRangeException {

  StructureMembers members = sdata.getStructureMembers();

  // loop over members
  for (Variable vm : s.getVariables()) {
    StructureMembers.Member m = members.findMember(vm.getShortName());
    if (null == m)
      continue; // this means that the data is missing from the ArrayStructure

    // convert String member data into CHAR data
    Array data = sdata.getArray(m);
    if (data instanceof ArrayObject && vm.getDataType() == DataType.CHAR && vm.getRank() > 0) {
      int strlen = vm.getShape(vm.getRank() - 1);
      data = ArrayChar.makeFromStringArray((ArrayObject) data, strlen); // turn it into an ArrayChar
    }

    // layout of the destination
    N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) vm.getSPobject();
    long begin = vinfo.begin + recnum * header.recsize; // this assumes unlimited dimension
    Section memberSection = vm.getShapeAsSection();
    Layout layout = new LayoutRegular(begin, vm.getElementSize(), vm.getShape(), memberSection);

    try {
      writeData(data, layout, vm.getDataType());
    } catch (Exception e) {
      log.error("Error writing member=" + vm.getShortName() + " in struct=" + s.getFullName(), e);
      throw new IOException(e);
    }
  }
}
 

public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
    return (vinfo != null) ? vinfo.toString() : "";
  }
  return null;
}
 

public void deflate(Formatter f, Variable v) {
  H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
  DataBTree btree = vinfo.btree;
  if (btree == null || vinfo.useFillValue) {
    f.format("%s not chunked%n", v.getShortName());
  }
}
 

public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
    return (vinfo != null) ? vinfo.toString() : "";
  }
  return null;
}
 

@Override
public void writeData(Variable v2, Section section, Array values) throws java.io.IOException, InvalidRangeException {
  N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) v2.getSPobject();
  DataType dataType = v2.getDataType();

  if (v2.isUnlimited()) {
    Range firstRange = section.getRange(0);
    setNumrecs(firstRange.last() + 1);
  }

  if (v2 instanceof Structure) {
    if (!(values instanceof ArrayStructure))
      throw new IllegalArgumentException("writeData for Structure: data must be ArrayStructure");

    if (v2.getRank() == 0)
      throw new IllegalArgumentException("writeData for Structure: must have rank > 0");

    Dimension d = v2.getDimension(0);
    if (!d.isUnlimited())
      throw new IllegalArgumentException("writeData for Structure: must have unlimited dimension");

    writeRecordData((Structure) v2, section, (ArrayStructure) values);

  } else {
    Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
        : new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
    writeData(values, layout, dataType);
  }
}
 

public void deflate(Formatter f, Variable v) {
  H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
  DataBTree btree = vinfo.btree;
  if (btree == null || vinfo.useFillValue) {
    f.format("%s not chunked%n", v.getShortName());
  }
}
 

@Override
public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
    return vinfo.toString();
  }
  return null;
}
 

/**
 * Read data from record structure. For N3, this is the only possible structure, and there can be no nesting.
 * Read all variables for each record, put in ByteBuffer.
 *
 * @param s the record structure
 * @param section the record range to read
 * @return an ArrayStructure, with all the data read in.
 * @throws IOException on error
 */
private ucar.ma2.Array readRecordData(ucar.nc2.Structure s, Section section) throws java.io.IOException {
  // has to be 1D
  Range recordRange = section.getRange(0);

  // create the ArrayStructure
  StructureMembers members = s.makeStructureMembers();
  for (StructureMembers.Member m : members.getMembers()) {
    Variable v2 = s.findVariable(m.getName());
    Vinfo vinfo = (Vinfo) v2.getSPobject();
    m.setDataParam((int) (vinfo.begin - header.recStart));
  }

  // protect against too large of reads
  if (header.recsize > Integer.MAX_VALUE)
    throw new IllegalArgumentException("Cant read records when recsize > " + Integer.MAX_VALUE);
  long nrecs = section.computeSize();
  if (nrecs * header.recsize > Integer.MAX_VALUE)
    throw new IllegalArgumentException(
        "Too large read: nrecs * recsize= " + (nrecs * header.recsize) + "bytes exceeds " + Integer.MAX_VALUE);

  members.setStructureSize((int) header.recsize);
  ArrayStructureBB structureArray = new ArrayStructureBB(members, new int[] {recordRange.length()});

  // loop over records
  byte[] result = structureArray.getByteBuffer().array();
  int count = 0;
  for (int recnum : recordRange) {
    if (debugRecord)
      System.out.println(" read record " + recnum);
    raf.seek(header.recStart + recnum * header.recsize); // where the record starts

    if (recnum != header.numrecs - 1) {
      raf.readFully(result, (int) (count * header.recsize), (int) header.recsize);
    } else {
      // "wart" allows file to be one byte short. since its always padding, we allow
      raf.read(result, (int) (count * header.recsize), (int) header.recsize);
    }
    count++;
  }

  return structureArray;
}
 

@Override
public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask)
    throws IOException, InvalidRangeException {
  FmrcInvLite.Gridset.Grid gridLite = (FmrcInvLite.Gridset.Grid) mainv.getSPobject();

  // read the original type - if its been promoted to a new type, the conversion happens after this read
  DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

  Array allData = Array.factory(dtype, section.getShape());
  int destPos = 0;

  // assumes the first two dimensions are runtime and time: LOOK: ensemble ??
  List<Range> ranges = section.getRanges();
  Range runRange = ranges.get(0);
  Range timeRange = ranges.get(1);
  List<Range> innerSection = ranges.subList(2, ranges.size());

  // keep track of open file - must be local variable for thread safety
  HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();
  try {

    // iterate over the desired runs
    for (int runIdx : runRange) {
      // Date runDate = vstate.runTimes.get(runIdx);

      // iterate over the desired forecast times
      for (int timeIdx : timeRange) {
        Array result = null;

        // find the inventory for this grid, runtime, and hour
        TimeInventory.Instance timeInv = gridLite.getInstance(runIdx, timeIdx);
        if (timeInv != null) {
          if (debugRead)
            System.out.printf("HIT %d %d ", runIdx, timeIdx);
          result = read(timeInv, gridLite.name, innerSection, openFilesRead); // may return null
          result = MAMath.convert(result, dtype); // just in case it need to be converted
        }

        // missing data
        if (result == null) {
          int[] shape = new Section(innerSection).getShape();
          result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
          if (debugRead)
            System.out.printf("MISS %d %d ", runIdx, timeIdx);
        }

        if (debugRead)
          System.out.printf("%d %d reallyRead %s %d bytes start at %d total size is %d%n", runIdx, timeIdx,
              mainv.getFullName(), result.getSize(), destPos, allData.getSize());

        Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
        destPos += result.getSize();
      }
    }
    return allData;

  } finally {
    // close any files used during this operation
    closeAll(openFilesRead);
  }
}
 

@Override
public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask)
    throws IOException, InvalidRangeException {
  Vstate1D vstate = (Vstate1D) mainv.getSPobject();

  // read the original type - if its been promoted to a new type, the conversion happens after this read
  DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

  Array allData = Array.factory(dtype, section.getShape());
  int destPos = 0;

  // assumes the first dimension is time: LOOK: what about ensemble ??
  List<Range> ranges = section.getRanges();
  Range timeRange = ranges.get(0);
  List<Range> innerSection = ranges.subList(1, ranges.size());

  // keep track of open files - must be local variable for thread safety
  HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();

  try {

    // iterate over the desired forecast times
    for (int timeIdx : timeRange) {
      Array result = null;

      // find the inventory for this grid, runtime, and hour
      TimeInventory.Instance timeInv = vstate.timeInv.getInstance(vstate.gridLite, timeIdx);
      if (timeInv == null) {
        if (logger.isDebugEnabled())
          logger.debug("Missing Inventory timeInx=" + timeIdx + " for " + mainv.getFullName() + " in "
              + state.lite.collectionName);
        // vstate.timeInv.getInstance(vstate.gridLite, timeIdx); // allow debugger
      } else if (timeInv.getDatasetLocation() != null) {
        if (debugRead)
          System.out.printf("HIT %s%n", timeInv);
        result = read(timeInv, mainv.getFullNameEscaped(), innerSection, openFilesRead); // may return null
        result = MAMath.convert(result, dtype); // just in case it need to be converted
      }

      // may have missing data
      if (result == null) {
        int[] shape = new Section(innerSection).getShape();
        result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
        if (debugRead)
          System.out.printf("MISS %d ", timeIdx);
      }

      if (debugRead)
        System.out.printf("%d reallyRead %s %d bytes start at %d total size is %d%n", timeIdx, mainv.getFullName(),
            result.getSize(), destPos, allData.getSize());

      Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
      destPos += result.getSize();
    }
    return allData;

  } finally {
    // close any files used during this operation
    closeAll(openFilesRead);
  }
}
 

public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
  H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject();
  if (debugRead)
    System.out.printf("%s read %s%n", v2.getFullName(), section);
  return readData(v2, vinfo.dataPos, section);
}
 

public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
  List<DataStorage> storage = (List<DataStorage>) v.getSPobject();
  ByteBuffer result = null;

  for (DataStorage dataStorage : storage) {
    if (dataStorage.isVlen)
      return readVlenData(v, section, dataStorage);

    if (dataStorage.sdata != null) {
      assert (v instanceof Structure);
      return readStructureData((Structure) v, section, dataStorage);
    }

    if (dataStorage.section.intersects(section)) { // LOOK WRONG
      raf.seek(dataStorage.filePos);
      byte[] data = new byte[dataStorage.size];
      raf.readFully(data);

      if (dataStorage.isDeflate) {
        ByteArrayInputStream bin = new ByteArrayInputStream(data);
        InflaterInputStream in = new InflaterInputStream(bin);
        ByteArrayOutputStream bout = new ByteArrayOutputStream(data.length * 7);
        IO.copy(in, bout);
        byte[] resultb = bout.toByteArray();
        result = ByteBuffer.wrap(resultb); // look - an extra copy !! override ByteArrayOutputStream to fix
        if (debug)
          System.out.printf(" uncompressedLen header=%d actual=%d%n", dataStorage.uncompressedLen, resultb.length);
        result.order(dataStorage.bo);

      } else {
        result = ByteBuffer.wrap(data);
        result.order(dataStorage.bo);
      }
    }
  }

  if (result == null)
    return null;

  return Array.factory(v.getDataType(), v.getShape(), result);
  // return dataArray.sectionNoReduce(section.getRanges());
}
 

public VarBean(Variable v) {
  this.v = v;
  this.vinfo = (H5header.Vinfo) v.getSPobject();
}
 

public VarBean(Variable v) {
  this.v = v;
  this.vinfo = (H5headerNew.Vinfo) v.getSPobject();
}