Java Code Examples for ucar.unidata.io.RandomAccessFile#acquire()

@Override
public boolean readChildren(Path indexFile, AddChildCallback callback) throws IOException {
  logger.debug("GribCdmIndex.readChildren {}", indexFile);
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
    GribCollectionType type = getType(raf);
    if (type == GribCollectionType.Partition1 || type == GribCollectionType.Partition2) {
      if (openIndex(raf, logger)) {
        String topDir = gribCollectionIndex.getTopDir();
        int n = gribCollectionIndex.getMfilesCount(); // partition index files stored in MFiles
        for (int i = 0; i < n; i++) {
          GribCollectionProto.MFile mfilep = gribCollectionIndex.getMfiles(i);
          callback.addChild(topDir, mfilep.getFilename(), mfilep.getLastModified());
        }
        return true;
      }
    }
    return false;
  }
}
 

@Override
public boolean readMFiles(Path indexFile, List<MFile> result) throws IOException {
  logger.debug("GribCdmIndex.readMFiles {}", indexFile);
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
    // GribCollectionType type = getType(raf);
    // if (type == GribCollectionType.GRIB1 || type == GribCollectionType.GRIB2) {
    if (openIndex(raf, logger)) {
      File protoDir = new File(gribCollectionIndex.getTopDir());
      int n = gribCollectionIndex.getMfilesCount();
      for (int i = 0; i < n; i++) {
        GribCollectionProto.MFile mfilep = gribCollectionIndex.getMfiles(i);
        result.add(new GcMFile(protoDir, mfilep.getFilename(), mfilep.getLastModified(), mfilep.getLength(),
            mfilep.getIndex()));
      }
    }
    return true;
    // }
  }
  // return false;
}
 

/**
 * Get the data file to use (if this is a template)
 *
 * @param eIndex ensemble index
 * @param tIndex time index
 * @return the current file
 * @throws IOException couldn't open the current file
 */
private RandomAccessFile getDataFile(int eIndex, int tIndex) throws IOException {

  String dataFilePath = gradsDDF.getFileName(eIndex, tIndex);
  if (!gradsDDF.isTemplate()) { // we only have one file
    if (dataFile != null) {
      return dataFile;
    }
  }
  if (dataFile != null) {
    String path = dataFile.getLocation();
    if (path.equals(dataFilePath)) {
      return dataFile;
    } else {
      dataFile.close();
    }
  }
  dataFile = RandomAccessFile.acquire(dataFilePath);
  dataFile.order(getByteOrder());
  return dataFile;
}
 

/**
 * Open an existing file (read only), specifying which IOSP is to be used.
 *
 * @param location location of file
 * @param iospClassName fully qualified class name of the IOSP class to handle this file
 * @param bufferSize RandomAccessFile buffer size, if <= 0, use default size
 * @param cancelTask allow task to be cancelled; may be null.
 * @param iospMessage special iosp tweaking (sent before open is called), may be null
 * @return NetcdfFile object, or null if cant find IOServiceProver
 * @throws IOException if read error
 * @throws ClassNotFoundException cannat find iospClassName in thye class path
 * @throws InstantiationException if class cannot be instantiated
 * @throws IllegalAccessException if class is not accessible
 * @deprecated use NetcdfFiles.open
 */
@Deprecated
public static NetcdfFile open(String location, String iospClassName, int bufferSize, CancelTask cancelTask,
    Object iospMessage) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException {

  Class iospClass = NetcdfFile.class.getClassLoader().loadClass(iospClassName);
  IOServiceProvider spi = (IOServiceProvider) iospClass.newInstance(); // fail fast

  // send iospMessage before iosp is opened
  if (iospMessage != null)
    spi.sendIospMessage(iospMessage);

  if (bufferSize <= 0)
    bufferSize = default_buffersize;

  RandomAccessFile raf = RandomAccessFile.acquire(canonicalizeUriString(location), bufferSize);

  NetcdfFile result = new NetcdfFile(spi, raf, location, cancelTask);

  // send after iosp is opened
  if (iospMessage != null)
    spi.sendIospMessage(iospMessage);

  return result;
}
 

@Test
public void testReadSubsection() throws IOException, InvalidRangeException {
  String location = canonjoin(TestDir.cdmTestDataDir, "thredds/public/testdata/nc_test_cdf5.nc");
  try (RandomAccessFile raf = RandomAccessFile.acquire(location)) {
    // Verify that this is a netcdf-5 file
    int format = NCheader.checkFileType(raf);
    Assert.assertTrue("Fail: file format is not CDF-5", format == NCheader.NC_FORMAT_64BIT_DATA);
  }
  try (NetcdfFile jni = openJni(location)) {
    jni.setLocation(location + " (jni)");
    Array data = read(jni, "f4", "0:2");
    if (prop_visual) {
      String dump = Ncdump.printArray(data);
      logger.debug(dump);
      String testresult = dump.replace('r', ' ').replace('\n', ' ').trim();
      visual("CDF Read", testresult);
    }
    Assert.assertTrue(String.format("***Fail: data mismatch"), MAMath.nearlyEquals(data, BASELINE));
    System.err.println("***Pass");
  }
}
 

@Override
public boolean isPartition(Path indexFile) throws IOException {
  logger.debug("GribCdmIndex.isPartition {}", indexFile);
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
    GribCollectionType type = getType(raf);
    return (type == GribCollectionType.Partition1) || (type == GribCollectionType.Partition2);
  }
}
 

@Override
public void open(RandomAccessFile raff, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  String dataFile = raff.getLocation();
  int pos = dataFile.lastIndexOf(".");
  String base = dataFile.substring(0, pos);
  String ext = dataFile.substring(pos);

  if (ext.equals(IDX_EXT)) {
    dataRaf = RandomAccessFile.acquire(base + DAT_EXT);
    stnRaf = RandomAccessFile.acquire(base + STN_EXT);

  } else if (ext.equals(DAT_EXT)) {
    dataRaf = raff;
    stnRaf = RandomAccessFile.acquire(base + STN_EXT);

  } else {
    stnRaf = raff;
    dataRaf = RandomAccessFile.acquire(base + DAT_EXT);
  }

  NcmlConstructor ncmlc = new NcmlConstructor();
  if (!ncmlc.populateFromResource("resources/nj22/iosp/ghcnm.ncml", ncfile)) {
    throw new IllegalStateException(ncmlc.getErrlog().toString());
  }
  ncfile.finish();

  dataVinfo = setVinfo(dataRaf, ncfile, dataPattern, "all_data");
  stnVinfo = setVinfo(stnRaf, ncfile, stnPattern, "station");

  StructureMembers.Member m = stnVinfo.sm.findMember(STNID);
  StructureDataRegexp.VinfoField f = (StructureDataRegexp.VinfoField) m.getDataObject();
  stn_fldno = f.fldno;

  // make index file if needed
  File idxFile = new File(base + IDX_EXT);
  if (!idxFile.exists())
    makeIndex(stnVinfo, dataVinfo, idxFile);
  else
    readIndex(idxFile.getPath());
}
 

/**
 * This is can only be used for local netcdf-3 files.
 *
 * @param filename location
 * @throws IOException if error
 * @deprecated use NetcdfFiles.open( location) or NetcdfDatasets.openFile( location)
 */
@Deprecated
public NetcdfFile(String filename) throws IOException {
  this.location = filename;
  RandomAccessFile raf = RandomAccessFile.acquire(filename);
  // ucar.unidata.io.RandomAccessFile raf = new ucar.unidata.io.MMapRandomAccessFile(filename, "r");
  this.iosp = SPFactory.getServiceProvider();
  iosp.open(raf, this, null);
  finish();
}
 

/**
 * Test if the file can be opened.
 *
 * @param fileName The file name.
 * @return Can be opened or not.
 * @throws IOException
 */
public static boolean canOpen(String fileName) throws IOException {
    boolean r = NetcdfFiles.canOpen(fileName);
    if (!r) {
        RandomAccessFile raf = RandomAccessFile.acquire(fileName);
        r = H5header.isValidFile(raf);
    }
    return r;
}
 

public static BufrCdmIndex readIndex(String indexFilename) throws IOException {
  BufrCdmIndex index = new BufrCdmIndex();
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFilename)) {
    index.readIndex(raf);
  }
  return index;
}
 

public void write() throws IOException {

    try (RandomAccessFile raf = RandomAccessFile.acquire(fileIn)) {

      Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
      while (scanner.hasNext()) {
        Grib2Record gr = scanner.next();
        float[] data = gr.readData(raf);
        for (int i = 0; i < data.length; i++) {
          data[i] = bitShave(data[i], mask11);
        }
      }
    }
  }
 

@Test
public void testCheckFileType() throws Exception {
  String location = canonjoin(TestDir.cdmTestDataDir, canonjoin(PREFIX, filename));
  try (RandomAccessFile raf = RandomAccessFile.acquire(location)) {
    // Verify type
    int found = NCheader.checkFileType(raf);
    String foundname = NCheader.formatName(found);
    String kindname = NCheader.formatName(kind);
    System.err.println("Testing format: " + kindname);
    Assert.assertTrue(String.format("***Fail: expected=%s found=%s%n", kindname, foundname), kind == found);
  }
}
 

public void reacquire() throws IOException {
  raf = RandomAccessFile.acquire(location);
  this.raf.order(rafOrder);
}
 

public synchronized void readRecords() throws IOException {
  if (this.sa != null)
    return;

  if (recordsLen == 0)
    return;
  byte[] b = new byte[recordsLen];

  try (RandomAccessFile indexRaf = RandomAccessFile.acquire(indexFilename)) {

    indexRaf.seek(recordsPos);
    indexRaf.readFully(b);

    /*
     * message SparseArray {
     * repeated uint32 size = 2 [packed=true]; // multidim sizes = shape[]
     * repeated uint32 track = 3 [packed=true]; // 1-based index into record list, 0 == missing
     * repeated Record records = 4; // List<Record>
     * uint32 ndups = 5; // duplicates found when creating
     * }
     */
    GribCollectionProto.SparseArray proto = GribCollectionProto.SparseArray.parseFrom(b);

    int nsizes = proto.getSizeCount();
    int[] size = new int[nsizes];
    for (int i = 0; i < nsizes; i++)
      size[i] = proto.getSize(i);

    int ntrack = proto.getTrackCount();
    int[] track = new int[ntrack];
    for (int i = 0; i < ntrack; i++)
      track[i] = proto.getTrack(i);

    int n = proto.getRecordsCount();
    List<Record> records = new ArrayList<>(n);
    for (int i = 0; i < n; i++) {
      GribCollectionProto.Record pr = proto.getRecords(i);
      records.add(new Record(pr.getFileno(), pr.getStartPos(), pr.getBmsOffset(), pr.getDrsOffset()));
    }
    int ndups = proto.getNdups();
    this.sa = new SparseArray<>(size, track, records, ndups);

  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
    logger.error(" file={} recordsLen={} recordPos={}", indexFilename, recordsLen, recordsPos);
    throw e;
  }
}
 

@Nullable
public static GribCollectionMutable openMutableGCFromIndex(String indexFilename, FeatureCollectionConfig config,
    boolean dataOnly, boolean useCache, Logger logger) {
  File indexFileInCache = useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : new File(indexFilename);
  if (indexFileInCache == null) {
    return null;
  }
  String indexFilenameInCache = indexFileInCache.getPath();
  String name = makeNameFromIndexFilename(indexFilename);
  GribCollectionMutable result = null;

  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFilenameInCache)) {
    GribCollectionType type = getType(raf);

    switch (type) {
      case GRIB2:
        result = Grib2CollectionBuilderFromIndex.openMutableGCFromIndex(name, raf, config, logger);
        break;
      case Partition2:
        result = Grib2PartitionBuilderFromIndex.openMutablePCFromIndex(name, raf, config, logger);
        break;
      case GRIB1:
        result = Grib1CollectionBuilderFromIndex.openMutableGCFromIndex(name, raf, config, logger);
        break;
      case Partition1:
        result = Grib1PartitionBuilderFromIndex.openMutablePCFromIndex(name, raf, config, logger);
        break;
      default:
        logger.warn("GribCdmIndex.openMutableGCFromIndex failed on {} type={}", indexFilenameInCache, type);
    }

    if (result != null) {
      result.lastModified = raf.getLastModified();
      result.fileSize = raf.length();
    }

  } catch (Throwable t) {
    logger.warn("GribCdmIndex.openMutableGCFromIndex failed on " + indexFilenameInCache, t);
  }

  if (result == null) {
    RandomAccessFile.eject(indexFilenameInCache);
    if (!indexFileInCache.delete())
      logger.warn("failed to delete {}", indexFileInCache.getPath());
  }

  return result;
}
 

@Nullable
public static GribCollectionImmutable openCdmIndex(String indexFilename, FeatureCollectionConfig config,
    boolean useCache, Logger logger) throws IOException {
  File indexFileInCache = useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : new File(indexFilename);
  if (indexFileInCache == null)
    return null;
  String indexFilenameInCache = indexFileInCache.getPath();
  String name = makeNameFromIndexFilename(indexFilename);
  GribCollectionImmutable result = null;

  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFilenameInCache)) {
    GribCollectionType type = getType(raf);

    switch (type) {
      case GRIB2:
        result = Grib2CollectionBuilderFromIndex.readFromIndex(name, raf, config, logger);
        break;
      case Partition2:
        result = Grib2PartitionBuilderFromIndex.createTimePartitionFromIndex(name, raf, config, logger);
        break;
      case GRIB1:
        result = Grib1CollectionBuilderFromIndex.readFromIndex(name, raf, config, logger);
        break;
      case Partition1:
        result = Grib1PartitionBuilderFromIndex.createTimePartitionFromIndex(name, raf, config, logger);
        break;
      default:
        logger.warn("GribCdmIndex.openCdmIndex failed on {} type={}", indexFilenameInCache, type);
    }
  } catch (FileNotFoundException ioe) {
    throw ioe;

  } catch (Throwable t) {
    logger.warn("GribCdmIndex.openCdmIndex failed on " + indexFilenameInCache, t);
    RandomAccessFile.eject(indexFilenameInCache);
    if (!indexFileInCache.delete())
      logger.warn("failed to delete {}", indexFileInCache.getPath());
  }

  return result;
}
 

private static RandomAccessFile getRaf(String location, int buffer_size) throws IOException {

    String uriString = location.trim();

    if (buffer_size <= 0)
      buffer_size = default_buffersize;

    RandomAccessFile raf;
    if (uriString.startsWith("http:") || uriString.startsWith("https:")) { // open through URL
      raf = new HTTPRandomAccessFile(uriString);

    } else if (uriString.startsWith("nodods:")) { // deprecated use httpserver
      uriString = "http" + uriString.substring(6);
      raf = new HTTPRandomAccessFile(uriString);

    } else if (uriString.startsWith("httpserver:")) { // open through URL
      uriString = "http" + uriString.substring(10);
      raf = new HTTPRandomAccessFile(uriString);

    } else if (uriString.startsWith("slurp:")) { // open through URL
      uriString = "http" + uriString.substring(5);
      byte[] contents = IO.readURLContentsToByteArray(uriString); // read all into memory
      raf = new InMemoryRandomAccessFile(uriString, contents);

    } else {
      // get rid of crappy microsnot \ replace with happy /
      uriString = StringUtil2.replace(uriString, '\\', "/");

      if (uriString.startsWith("file:")) {
        // uriString = uriString.substring(5);
        uriString = StringUtil2.unescape(uriString.substring(5)); // 11/10/2010 from [email protected]
      }

      String uncompressedFileName = null;
      try {
        stringLocker.control(uriString); // Avoid race condition where the decompressed file is trying to be read by one
                                         // thread while another is decompressing it
        uncompressedFileName = makeUncompressed(uriString);
      } catch (Exception e) {
        log.warn("Failed to uncompress {}, err= {}; try as a regular file.", uriString, e.getMessage());
        // allow to fall through to open the "compressed" file directly - may be a misnamed suffix
      } finally {
        stringLocker.release(uriString);
      }

      if (uncompressedFileName != null) {
        // open uncompressed file as a RandomAccessFile.
        raf = RandomAccessFile.acquire(uncompressedFileName, buffer_size);
        // raf = new ucar.unidata.io.MMapRandomAccessFile(uncompressedFileName, "r");

      } else {
        // normal case - not compressed
        raf = RandomAccessFile.acquire(uriString, buffer_size);
        // raf = new ucar.unidata.io.MMapRandomAccessFile(uriString, "r");
      }
    }

    return raf;
  }
 

@Override
public void open(RandomAccessFile raff, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  super.open(raff, ncfile, cancelTask);
  int pos = location.lastIndexOf(".");
  String ext = location.substring(pos);

  File file = new File(location);
  File stnFile = getStnFile(location);
  if (stnFile == null)
    throw new FileNotFoundException("Station File does not exist=" + location);

  if (ext.equals(IDX_EXT)) {
    stnRaf = RandomAccessFile.acquire(stnFile.getPath());

  } else if (ext.equals(DAT_EXT)) {
    stnRaf = RandomAccessFile.acquire(stnFile.getPath());
    dataRaf = raff;

    // extract the station id
    String name = file.getName();
    stationId = name.substring(0, name.length() - DAT_EXT.length());

  } else { // pointed to the station file
    stnRaf = raff;
    dataDir = new File(file.getParentFile(), DAT_DIR);
  }

  NcmlConstructor ncmlc = new NcmlConstructor();
  if (!ncmlc.populateFromResource("resources/nj22/iosp/igra-por.ncml", ncfile)) {
    throw new IllegalStateException(ncmlc.getErrlog().toString());
  }
  ncfile.finish();

  // dataVinfo = setVinfo(dataRaf, ncfile, dataPattern, "all_data");
  stnVinfo = setVinfo(stnRaf, ncfile, stnPattern, "station");
  seriesVinfo = setVinfo(stnRaf, ncfile, dataHeaderPattern, "station.time_series");
  profileVinfo = setVinfo(stnRaf, ncfile, dataPattern, "station.time_series.levels");

  StructureMembers.Member m = stnVinfo.sm.findMember(STNID);
  StructureDataRegexp.VinfoField f = (StructureDataRegexp.VinfoField) m.getDataObject();
  stn_fldno = f.fldno;

  /*
   * make index file if needed
   * File idxFile = new File(base + IDX_EXT);
   * if (!idxFile.exists())
   * makeIndex(stnVinfo, dataVinfo, idxFile);
   * else
   * readIndex(idxFile.getPath());
   */
}
 

public static long testValid(String ufilename) throws IOException {
  boolean lookForHeader = false;

  // gotta make it
  try (RandomAccessFile raf = RandomAccessFile.acquire(ufilename)) {
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    String test = raf.readString(8);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.startsWith("AR2V000")) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;

    while (!eof) {

      if (lookForHeader) {
        test = raf.readString(8);
        if (test.equals(Level2VolumeScan.ARCHIVE2) || test.startsWith("AR2V000")) {
          System.out.println("  found header= " + test);
          raf.skipBytes(16);
          lookForHeader = false;
        } else {
          raf.skipBytes(-8);
        }
      }

      try {
        numCompBytes = raf.readInt();
        if (numCompBytes == -1) {
          System.out.println("\n--done: numCompBytes=-1 ");
          break;
        }
      } catch (EOFException ee) {
        System.out.println("\n--got EOFException ");
        break; // assume this is ok
      }

      System.out.print(" " + numCompBytes + ",");
      if (numCompBytes < 0) {
        System.out.println("\n--last block " + numCompBytes);
        numCompBytes = -numCompBytes;
        if (!lookForHeader) {
          eof = true;
        }
      }

      raf.skipBytes(numCompBytes);
    }
    return raf.getFilePointer();
  } catch (EOFException e) {
    e.printStackTrace();
  }
  return 0;
}
 

/**
 * Create a McIDASGrid Reader from the file
 *
 * @param filename filename
 * @throws IOException problem reading file
 */
public McIDASGridReader(String filename) throws IOException {
  this(RandomAccessFile.acquire(filename));
}