Java Code Examples for ucar.nc2.Variable#getSPobject()

The following examples show how to use ucar.nc2.Variable#getSPobject() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Read the data for each variable passed in
 */
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {

  // subset
  Object data;
  Array outputData;
  byte[] vdata;
  NOWRadheader.Vinfo vinfo;
  ByteBuffer bos;
  List<Range> ranges = section.getRanges();

  vinfo = (NOWRadheader.Vinfo) v2.getSPobject();
  vdata = headerParser.getData((int) vinfo.hoff);

  bos = ByteBuffer.wrap(vdata);
  data = readOneScanData(bos, vinfo, v2.getShortName());
  outputData = Array.factory(v2.getDataType(), v2.getShape(), data);
  outputData = outputData.flip(1);

  // outputData = outputData.flip(2);
  return (outputData.sectionNoReduce(ranges).copy());

  // return outputData;
}
 
Example 2
public Array readData(Variable v2, Section section) throws IOException {
  Vgroup vgroup = (Vgroup) v2.getSPobject();

  Range scanRange = section.getRange(0);
  Range radialRange = section.getRange(1);
  Range gateRange = section.getRange(2);

  Array data = Array.factory(v2.getDataType(), section.getShape());
  IndexIterator ii = data.getIndexIterator();

  for (int scanIdx : scanRange) {
    Ray[] mapScan = vgroup.map[scanIdx];
    readOneScan(mapScan, radialRange, gateRange, vgroup.abbrev, ii);
  }

  return data;
}
 
Example 3
private StructureData readStructure(Structure s, ArrayStructureW asw, long dataPos)
    throws IOException, InvalidRangeException {
  StructureDataW sdata = new StructureDataW(asw.getStructureMembers());
  if (debug)
    System.out.println(" readStructure " + s.getFullName() + " dataPos = " + dataPos);

  for (Variable v2 : s.getVariables()) {
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject();
    if (debug)
      System.out.println(" readStructureMember " + v2.getFullName() + " vinfo = " + vinfo);
    Array dataArray = readData(v2, dataPos + vinfo.dataPos, v2.getShapeAsSection());
    sdata.setMemberData(v2.getShortName(), dataArray);
  }

  return sdata;
}
 
Example 4
public void showCompress(Formatter f) throws IOException {
  Size totalSize = new Size(0, 0);
  for (Variable v : ncfile.getVariables()) {
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
    showCompress(v, vinfo, totalSize, f);
  }
  f.format("%n");
  f.format(" total bytes   = %d%n", totalSize.nominal);
  f.format(" total storage = %d%n", totalSize.storage);
  f.format(" compression = %f%n", totalSize.getRatio());
  f.format(" nchunks     = %d%n", totalSize.count);

  File raf = new File(ncfile.getLocation());
  f.format(" file size    = %d%n", raf.length());
  float overhead = totalSize.storage == 0 ? 0 : ((float) raf.length() / totalSize.storage);
  f.format(" overhead     = %f%n", overhead);
}
 
Example 5
public void showCompress(Formatter f) throws IOException {
  NetcdfFile ncfile = iosp.getNetcdfFile();

  Size totalSize = new Size(0, 0);
  for (Variable v : ncfile.getVariables()) {
    H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
    showCompress(v, vinfo, totalSize, f);
  }
  f.format("%n");
  f.format(" total bytes   = %d%n", totalSize.nominal);
  f.format(" total storage = %d%n", totalSize.storage);
  f.format(" compression = %f%n", totalSize.getRatio());
  f.format(" nchunks     = %d%n", totalSize.count);

  File raf = new File(ncfile.getLocation());
  f.format(" file size    = %d%n", raf.length());
  float overhead = totalSize.storage == 0 ? 0 : ((float) raf.length() / totalSize.storage);
  f.format(" overhead     = %f%n", overhead);
}
 
Example 6
@Override
public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
    return vinfo.toString();
  }
  return null;
}
 
Example 7
public void deflate(Formatter f, Variable v) {
  H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v.getSPobject();
  DataBTree btree = vinfo.btree;
  if (btree == null || vinfo.useFillValue) {
    f.format("%s not chunked%n", v.getShortName());
  }
}
 
Example 8
@Override
public void writeData(Variable v2, Section section, Array values) throws java.io.IOException, InvalidRangeException {
  N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) v2.getSPobject();
  DataType dataType = v2.getDataType();

  if (v2.isUnlimited()) {
    Range firstRange = section.getRange(0);
    setNumrecs(firstRange.last() + 1);
  }

  if (v2 instanceof Structure) {
    if (!(values instanceof ArrayStructure))
      throw new IllegalArgumentException("writeData for Structure: data must be ArrayStructure");

    if (v2.getRank() == 0)
      throw new IllegalArgumentException("writeData for Structure: must have rank > 0");

    Dimension d = v2.getDimension(0);
    if (!d.isUnlimited())
      throw new IllegalArgumentException("writeData for Structure: must have unlimited dimension");

    writeRecordData((Structure) v2, section, (ArrayStructure) values);

  } else {
    Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
        : new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
    writeData(values, layout, dataType);
  }
}
 
Example 9
private void writeRecordData(ucar.nc2.Structure s, int recnum, StructureData sdata)
    throws java.io.IOException, ucar.ma2.InvalidRangeException {

  StructureMembers members = sdata.getStructureMembers();

  // loop over members
  for (Variable vm : s.getVariables()) {
    StructureMembers.Member m = members.findMember(vm.getShortName());
    if (null == m)
      continue; // this means that the data is missing from the ArrayStructure

    // convert String member data into CHAR data
    Array data = sdata.getArray(m);
    if (data instanceof ArrayObject && vm.getDataType() == DataType.CHAR && vm.getRank() > 0) {
      int strlen = vm.getShape(vm.getRank() - 1);
      data = ArrayChar.makeFromStringArray((ArrayObject) data, strlen); // turn it into an ArrayChar
    }

    // layout of the destination
    N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) vm.getSPobject();
    long begin = vinfo.begin + recnum * header.recsize; // this assumes unlimited dimension
    Section memberSection = vm.getShapeAsSection();
    Layout layout = new LayoutRegular(begin, vm.getElementSize(), vm.getShape(), memberSection);

    try {
      writeData(data, layout, vm.getDataType());
    } catch (Exception e) {
      log.error("Error writing member=" + vm.getShortName() + " in struct=" + s.getFullName(), e);
      throw new IOException(e);
    }
  }
}
 
Example 10
public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
    return (vinfo != null) ? vinfo.toString() : "";
  }
  return null;
}
 
Example 11
public void deflate(Formatter f, Variable v) {
  H5header.Vinfo vinfo = (H5header.Vinfo) v.getSPobject();
  DataBTree btree = vinfo.btree;
  if (btree == null || vinfo.useFillValue) {
    f.format("%s not chunked%n", v.getShortName());
  }
}
 
Example 12
public String toStringDebug(Object o) {
  if (o instanceof Variable) {
    Variable v = (Variable) o;
    H4header.Vinfo vinfo = (H4header.Vinfo) v.getSPobject();
    return (vinfo != null) ? vinfo.toString() : "";
  }
  return null;
}
 
Example 13
public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
  List<DataStorage> storage = (List<DataStorage>) v.getSPobject();
  ByteBuffer result = null;

  for (DataStorage dataStorage : storage) {
    if (dataStorage.isVlen)
      return readVlenData(v, section, dataStorage);

    if (dataStorage.sdata != null) {
      assert (v instanceof Structure);
      return readStructureData((Structure) v, section, dataStorage);
    }

    if (dataStorage.section.intersects(section)) { // LOOK WRONG
      raf.seek(dataStorage.filePos);
      byte[] data = new byte[dataStorage.size];
      raf.readFully(data);

      if (dataStorage.isDeflate) {
        ByteArrayInputStream bin = new ByteArrayInputStream(data);
        InflaterInputStream in = new InflaterInputStream(bin);
        ByteArrayOutputStream bout = new ByteArrayOutputStream(data.length * 7);
        IO.copy(in, bout);
        byte[] resultb = bout.toByteArray();
        result = ByteBuffer.wrap(resultb); // look - an extra copy !! override ByteArrayOutputStream to fix
        if (debug)
          System.out.printf(" uncompressedLen header=%d actual=%d%n", dataStorage.uncompressedLen, resultb.length);
        result.order(dataStorage.bo);

      } else {
        result = ByteBuffer.wrap(data);
        result.order(dataStorage.bo);
      }
    }
  }

  if (result == null)
    return null;

  return Array.factory(v.getDataType(), v.getShape(), result);
  // return dataArray.sectionNoReduce(section.getRanges());
}
 
Example 14
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
  H5headerNew.Vinfo vinfo = (H5headerNew.Vinfo) v2.getSPobject();
  if (debugRead)
    System.out.printf("%s read %s%n", v2.getFullName(), section);
  return readData(v2, vinfo.dataPos, section);
}
 
Example 15
/**
 * Read data from record structure. For N3, this is the only possible structure, and there can be no nesting.
 * Read all variables for each record, put in ByteBuffer.
 *
 * @param s the record structure
 * @param section the record range to read
 * @return an ArrayStructure, with all the data read in.
 * @throws IOException on error
 */
private ucar.ma2.Array readRecordData(ucar.nc2.Structure s, Section section) throws java.io.IOException {
  // has to be 1D
  Range recordRange = section.getRange(0);

  // create the ArrayStructure
  StructureMembers members = s.makeStructureMembers();
  for (StructureMembers.Member m : members.getMembers()) {
    Variable v2 = s.findVariable(m.getName());
    Vinfo vinfo = (Vinfo) v2.getSPobject();
    m.setDataParam((int) (vinfo.begin - header.recStart));
  }

  // protect against too large of reads
  if (header.recsize > Integer.MAX_VALUE)
    throw new IllegalArgumentException("Cant read records when recsize > " + Integer.MAX_VALUE);
  long nrecs = section.computeSize();
  if (nrecs * header.recsize > Integer.MAX_VALUE)
    throw new IllegalArgumentException(
        "Too large read: nrecs * recsize= " + (nrecs * header.recsize) + "bytes exceeds " + Integer.MAX_VALUE);

  members.setStructureSize((int) header.recsize);
  ArrayStructureBB structureArray = new ArrayStructureBB(members, new int[] {recordRange.length()});

  // loop over records
  byte[] result = structureArray.getByteBuffer().array();
  int count = 0;
  for (int recnum : recordRange) {
    if (debugRecord)
      System.out.println(" read record " + recnum);
    raf.seek(header.recStart + recnum * header.recsize); // where the record starts

    if (recnum != header.numrecs - 1) {
      raf.readFully(result, (int) (count * header.recsize), (int) header.recsize);
    } else {
      // "wart" allows file to be one byte short. since its always padding, we allow
      raf.read(result, (int) (count * header.recsize), (int) header.recsize);
    }
    count++;
  }

  return structureArray;
}
 
Example 16
@Override
public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask)
    throws IOException, InvalidRangeException {
  FmrcInvLite.Gridset.Grid gridLite = (FmrcInvLite.Gridset.Grid) mainv.getSPobject();

  // read the original type - if its been promoted to a new type, the conversion happens after this read
  DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

  Array allData = Array.factory(dtype, section.getShape());
  int destPos = 0;

  // assumes the first two dimensions are runtime and time: LOOK: ensemble ??
  List<Range> ranges = section.getRanges();
  Range runRange = ranges.get(0);
  Range timeRange = ranges.get(1);
  List<Range> innerSection = ranges.subList(2, ranges.size());

  // keep track of open file - must be local variable for thread safety
  HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();
  try {

    // iterate over the desired runs
    for (int runIdx : runRange) {
      // Date runDate = vstate.runTimes.get(runIdx);

      // iterate over the desired forecast times
      for (int timeIdx : timeRange) {
        Array result = null;

        // find the inventory for this grid, runtime, and hour
        TimeInventory.Instance timeInv = gridLite.getInstance(runIdx, timeIdx);
        if (timeInv != null) {
          if (debugRead)
            System.out.printf("HIT %d %d ", runIdx, timeIdx);
          result = read(timeInv, gridLite.name, innerSection, openFilesRead); // may return null
          result = MAMath.convert(result, dtype); // just in case it need to be converted
        }

        // missing data
        if (result == null) {
          int[] shape = new Section(innerSection).getShape();
          result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
          if (debugRead)
            System.out.printf("MISS %d %d ", runIdx, timeIdx);
        }

        if (debugRead)
          System.out.printf("%d %d reallyRead %s %d bytes start at %d total size is %d%n", runIdx, timeIdx,
              mainv.getFullName(), result.getSize(), destPos, allData.getSize());

        Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
        destPos += result.getSize();
      }
    }
    return allData;

  } finally {
    // close any files used during this operation
    closeAll(openFilesRead);
  }
}
 
Example 17
@Override
public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask)
    throws IOException, InvalidRangeException {
  Vstate1D vstate = (Vstate1D) mainv.getSPobject();

  // read the original type - if its been promoted to a new type, the conversion happens after this read
  DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType();

  Array allData = Array.factory(dtype, section.getShape());
  int destPos = 0;

  // assumes the first dimension is time: LOOK: what about ensemble ??
  List<Range> ranges = section.getRanges();
  Range timeRange = ranges.get(0);
  List<Range> innerSection = ranges.subList(1, ranges.size());

  // keep track of open files - must be local variable for thread safety
  HashMap<String, NetcdfDataset> openFilesRead = new HashMap<>();

  try {

    // iterate over the desired forecast times
    for (int timeIdx : timeRange) {
      Array result = null;

      // find the inventory for this grid, runtime, and hour
      TimeInventory.Instance timeInv = vstate.timeInv.getInstance(vstate.gridLite, timeIdx);
      if (timeInv == null) {
        if (logger.isDebugEnabled())
          logger.debug("Missing Inventory timeInx=" + timeIdx + " for " + mainv.getFullName() + " in "
              + state.lite.collectionName);
        // vstate.timeInv.getInstance(vstate.gridLite, timeIdx); // allow debugger
      } else if (timeInv.getDatasetLocation() != null) {
        if (debugRead)
          System.out.printf("HIT %s%n", timeInv);
        result = read(timeInv, mainv.getFullNameEscaped(), innerSection, openFilesRead); // may return null
        result = MAMath.convert(result, dtype); // just in case it need to be converted
      }

      // may have missing data
      if (result == null) {
        int[] shape = new Section(innerSection).getShape();
        result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values
        if (debugRead)
          System.out.printf("MISS %d ", timeIdx);
      }

      if (debugRead)
        System.out.printf("%d reallyRead %s %d bytes start at %d total size is %d%n", timeIdx, mainv.getFullName(),
            result.getSize(), destPos, allData.getSize());

      Array.arraycopy(result, 0, allData, destPos, (int) result.getSize());
      destPos += result.getSize();
    }
    return allData;

  } finally {
    // close any files used during this operation
    closeAll(openFilesRead);
  }
}
 
Example 18
public VarBean(Variable v) {
  this.v = v;
  this.vinfo = (H5header.Vinfo) v.getSPobject();
}
 
Example 19
public VarBean(Variable v) {
  this.v = v;
  this.vinfo = (H5headerNew.Vinfo) v.getSPobject();
}