Java Code Examples for ucar.nc2.Variable#getElementSize()

The following examples show how to use ucar.nc2.Variable#getElementSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GtopoIosp.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public Array readData(Variable v2, Section wantSection) throws IOException, InvalidRangeException {
  // raf.seek(0);
  raf.order(RandomAccessFile.BIG_ENDIAN);

  int size = (int) wantSection.computeSize();
  short[] arr = new short[size];

  LayoutRegular indexer = new LayoutRegular(0, v2.getElementSize(), v2.getShape(), wantSection);
  while (indexer.hasNext()) {
    Layout.Chunk chunk = indexer.next();
    raf.seek(chunk.getSrcPos());
    raf.readShort(arr, (int) chunk.getDestElem(), chunk.getNelems()); // copy into primitive array
  }

  return Array.factory(v2.getDataType(), wantSection.getShape(), arr);
}
 
Example 2
Source File: H5diagNew.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public void showCompress(Variable v, H5headerNew.Vinfo vinfo, Size total, Formatter f) throws IOException {
  H5objects.MessageDataspace mdt = vinfo.mds;

  long total_elems = 1;
  f.format("%8s %-40s(", v.getDataType(), v.getShortName());
  for (int len : mdt.dimLength) {
    f.format("%d ", len);
    total_elems *= len;
  }
  boolean sizeOk = total_elems == v.getSize();
  total_elems = v.getSize();

  long nominalSize = total_elems * v.getElementSize();

  Size size = new Size(nominalSize, 1);
  countStorageSize(vinfo, size);
  total.storage += size.storage;
  total.nominal += nominalSize;
  total.count += size.count;
  float ratio = (size.storage == 0) ? 0 : (float) nominalSize / size.storage;

  f.format(") == %d nelems %s == %d bytes storage = %d (%f) nchunks = %d%n", total_elems, sizeOk ? "" : "*",
      nominalSize, size.storage, ratio, size.count);
}
 
Example 3
Source File: H5diag.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public void showCompress(Variable v, H5header.Vinfo vinfo, Size total, Formatter f) throws IOException {
  H5header.MessageDataspace mdt = vinfo.mds;

  long total_elems = 1;
  f.format("%8s %-40s(", v.getDataType(), v.getShortName());
  for (int len : mdt.dimLength) {
    f.format("%d ", len);
    total_elems *= len;
  }
  boolean sizeOk = total_elems == v.getSize();
  total_elems = v.getSize();

  long nominalSize = total_elems * v.getElementSize();

  Size size = new Size(nominalSize, 1);
  countStorageSize(vinfo, size);
  total.storage += size.storage;
  total.nominal += nominalSize;
  total.count += size.count;
  float ratio = (size.storage == 0) ? 0 : (float) nominalSize / size.storage;

  f.format(") == %d nelems %s == %d bytes storage = %d (%f) nchunks = %d%n", total_elems, sizeOk ? "" : "*",
      nominalSize, size.storage, ratio, size.count);
}
 
Example 4
Source File: NcStream.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private static Builder encodeVar(Variable var, int sizeToCache) throws IOException {
  Builder builder = NcStreamProto.Variable.newBuilder();
  builder.setName(var.getShortName());
  builder.setDataType(convertDataType(var.getDataType()));
  if (var.getDataType().isEnum()) {
    EnumTypedef enumType = var.getEnumTypedef();
    if (enumType != null)
      builder.setEnumType(enumType.getShortName());
  }

  for (Dimension dim : var.getDimensions()) {
    builder.addShape(encodeDim(dim));
  }

  for (Attribute att : var.attributes()) {
    builder.addAtts(encodeAtt(att));
  }

  // put small amounts of data in header "immediate mode"
  if (var.isCaching() && var.getDataType().isNumeric()) {
    if (var.isCoordinateVariable() || var.getSize() * var.getElementSize() < sizeToCache) {
      Array data = var.read();
      ByteBuffer bb = data.getDataAsByteBuffer();
      builder.setData(ByteString.copyFrom(bb.array()));
    }
  }

  return builder;
}
 
Example 5
Source File: N3iospWriter.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void writeData(Variable v2, Section section, Array values) throws java.io.IOException, InvalidRangeException {
  N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) v2.getSPobject();
  DataType dataType = v2.getDataType();

  if (v2.isUnlimited()) {
    Range firstRange = section.getRange(0);
    setNumrecs(firstRange.last() + 1);
  }

  if (v2 instanceof Structure) {
    if (!(values instanceof ArrayStructure))
      throw new IllegalArgumentException("writeData for Structure: data must be ArrayStructure");

    if (v2.getRank() == 0)
      throw new IllegalArgumentException("writeData for Structure: must have rank > 0");

    Dimension d = v2.getDimension(0);
    if (!d.isUnlimited())
      throw new IllegalArgumentException("writeData for Structure: must have unlimited dimension");

    writeRecordData((Structure) v2, section, (ArrayStructure) values);

  } else {
    Layout layout = (!v2.isUnlimited()) ? new LayoutRegular(vinfo.begin, v2.getElementSize(), v2.getShape(), section)
        : new LayoutRegularSegmented(vinfo.begin, v2.getElementSize(), header.recsize, v2.getShape(), section);
    writeData(values, layout, dataType);
  }
}
 
Example 6
Source File: N3iospWriter.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private void writeRecordData(ucar.nc2.Structure s, int recnum, StructureData sdata)
    throws java.io.IOException, ucar.ma2.InvalidRangeException {

  StructureMembers members = sdata.getStructureMembers();

  // loop over members
  for (Variable vm : s.getVariables()) {
    StructureMembers.Member m = members.findMember(vm.getShortName());
    if (null == m)
      continue; // this means that the data is missing from the ArrayStructure

    // convert String member data into CHAR data
    Array data = sdata.getArray(m);
    if (data instanceof ArrayObject && vm.getDataType() == DataType.CHAR && vm.getRank() > 0) {
      int strlen = vm.getShape(vm.getRank() - 1);
      data = ArrayChar.makeFromStringArray((ArrayObject) data, strlen); // turn it into an ArrayChar
    }

    // layout of the destination
    N3headerNew.Vinfo vinfo = (N3headerNew.Vinfo) vm.getSPobject();
    long begin = vinfo.begin + recnum * header.recsize; // this assumes unlimited dimension
    Section memberSection = vm.getShapeAsSection();
    Layout layout = new LayoutRegular(begin, vm.getElementSize(), vm.getShape(), memberSection);

    try {
      writeData(data, layout, vm.getDataType());
    } catch (Exception e) {
      log.error("Error writing member=" + vm.getShortName() + " in struct=" + s.getFullName(), e);
      throw new IOException(e);
    }
  }
}
 
Example 7
Source File: Nc4ChunkingStrategyGrib.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public boolean isChunked(Variable v) {
  if (v.isUnlimited())
    return true;
  // if (getChunkAttribute(v) != null) return true;

  int n = v.getRank();
  return n >= 2 && v.getSize() * v.getElementSize() > getMinVariableSize();
}
 
Example 8
Source File: Nc4ChunkingDefault.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public boolean isChunked(Variable v) {
  if (v.isUnlimited())
    return true;
  // if (getChunkAttribute(v) != null) return true;

  long size = v.getSize() * v.getElementSize();
  return (size > minVariableSize);
}
 
Example 9
Source File: NetcdfCopier.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private void copyVariableData(NetcdfFormatWriter ncwriter, Group groupIn, Group groupOut, Count counter,
    CancelTask cancel) throws IOException {
  for (Variable oldVar : groupIn.getVariables()) {
    if (cancel.isCancel()) {
      break;
    }
    Variable newVar = groupOut.findVariableLocal(oldVar.getShortName());
    if (debug) {
      System.out.format("write var= %s size = %d type = %s%n", oldVar.getFullName(), oldVar.getSize(),
          oldVar.getDataType());
    }
    cancel.setProgress("writing " + oldVar.getFullName(), counter.countVars++);

    long size = oldVar.getSize() * oldVar.getElementSize();
    counter.bytes += size;

    if (size <= maxSize) {
      copyAll(ncwriter, oldVar, newVar);
    } else {
      copySome(ncwriter, oldVar, newVar, maxSize, cancel);
    }
  }

  for (Group nestedIn : groupIn.getGroups()) {
    if (cancel.isCancel()) {
      break;
    }
    Group nestedOut = groupOut.findGroupLocal(nestedIn.getShortName());
    copyVariableData(ncwriter, nestedIn, nestedOut, counter, cancel);
  }
}
 
Example 10
Source File: NetcdfFormatWriter.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private long calcSize(Group group) {
  long totalSizeOfVars = 0;
  for (Variable var : group.getVariables()) {
    totalSizeOfVars += Dimensions.getSize(var.getDimensions()) * var.getElementSize();
  }
  for (Group nested : group.getGroups()) {
    totalSizeOfVars += calcSize(nested);
  }
  return totalSizeOfVars;
}
 
Example 11
Source File: TestHttpOpen.java    From tds with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private long readAllData(NetcdfFile ncfile) throws IOException {
  System.out.println("------Open " + ncfile.getLocation());

  long total = 0;
  for (Variable v : ncfile.getVariables()) {
    long nbytes = v.getSize() * v.getElementSize();
    System.out.println("  Try to read variable " + v.getFullName() + " " + nbytes);
    v.read();
    total += v.getSize() * v.getElementSize();
  }
  ncfile.close();
  return total;
}
 
Example 12
Source File: CFGridWriter.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
private long processTransformationVars(ArrayList<Variable> varList, ArrayList<String> varNameList, NetcdfDataset ncd,
    ucar.nc2.dt.GridDataset gds, GridDatatype grid, Range timeRange, Range zRangeUse, Range yRange, Range xRange,
    int z_stride, int y_stride, int x_stride) throws InvalidRangeException {

  long varsSize = 0L;
  List<CoordinateTransform> cctt = grid.getCoordinateSystem().getCoordinateTransforms();
  for (CoordinateTransform ct : cctt) {
    Parameter param = ct.findParameterIgnoreCase(CF.FORMULA_TERMS);

    if (param != null) {
      String[] varStrings = param.getStringValue().split(" ");
      for (int i = 1; i < varStrings.length; i += 2) {
        Variable paramVar = ncd.findVariable(varStrings[i].trim());

        if (!varNameList.contains(varStrings[i]) && (null != paramVar)) {

          if (gds.findGridDatatype(paramVar.getFullName()) != null) {
            // Subset if needed
            if ((null != timeRange) || (zRangeUse != null) || (x_stride > 1 && y_stride > 1)
                || (yRange != null || xRange != null)) {
              GridDatatype complementaryGrid = gds.findGridDatatype(paramVar.getFullName());
              complementaryGrid = complementaryGrid.makeSubset(null, null, timeRange, zRangeUse, yRange, xRange);
              paramVar = complementaryGrid.getVariable();
            }
          } else {
            // Also have to subset the var if it is not a grid but has vertical dimension (the dimensionless vars in
            // the formula) and zRangeUse != null
            if (zRangeUse != null && paramVar.getRank() == 1) {
              List<Range> ranges = new ArrayList<Range>();
              ranges.add(zRangeUse);
              paramVar = paramVar.section(ranges);
            }
          }
          varNameList.add(paramVar.getFullName());
          varsSize += paramVar.getSize() * paramVar.getElementSize();
          varList.add(paramVar);
        }

      }
    }
  }

  return varsSize;

}
 
Example 13
Source File: NetcdfCopier.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
/**
 * Copies data from {@code oldVar} to {@code newVar}. The writes are done in a series of chunks no larger than
 * {@code maxChunkSize}
 * bytes.
 *
 * @param oldVar a variable from the original file to copy data from.
 * @param newVar a variable from the original file to copy data from.
 * @param maxChunkSize the size, <b>in bytes</b>, of the largest chunk to write.
 * @param cancel allow user to cancel, may be null.
 * @throws IOException if an I/O error occurs.
 */
private void copySome(NetcdfFormatWriter ncwriter, Variable oldVar, Variable newVar, long maxChunkSize,
    CancelTask cancel) throws IOException {
  long maxChunkElems = maxChunkSize / oldVar.getElementSize();
  long byteWriteTotal = 0;

  ChunkingIndex index = new ChunkingIndex(oldVar.getShape());
  while (index.currentElement() < index.getSize()) {
    try {
      int[] chunkOrigin = index.getCurrentCounter();
      int[] chunkShape = index.computeChunkShape(maxChunkElems);

      cancel.setProgress(
          "Reading chunk " + new Section(chunkOrigin, chunkShape) + " from variable: " + oldVar.getShortName(), -1);

      Array data = oldVar.read(chunkOrigin, chunkShape);
      if (!getOutputFormat().isNetdf4format() && oldVar.getDataType() == DataType.STRING) {
        data = convertDataToChar(newVar, data);
      }

      if (data.getSize() > 0) { // zero when record dimension = 0
        cancel.setProgress(
            "Writing chunk " + new Section(chunkOrigin, chunkShape) + " from variable: " + oldVar.getShortName(), -1);

        ncwriter.write(newVar, chunkOrigin, data);
        if (debugWrite) {
          System.out.println(" write " + data.getSize() + " bytes at " + new Section(chunkOrigin, chunkShape));
        }

        byteWriteTotal += data.getSize();
      }

      index.setCurrentCounter(index.currentElement() + (int) Index.computeSize(chunkShape));
      if (cancel.isCancel()) {
        return;
      }

    } catch (InvalidRangeException e) {
      e.printStackTrace();
      throw new IOException(e.getMessage());
    }
  }
}