/* * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.dt.point; import ucar.ma2.*; import ucar.nc2.*; import ucar.nc2.constants.FeatureType; import ucar.nc2.constants.AxisType; import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.util.CancelTask; import ucar.nc2.dt.*; import java.io.*; import java.util.*; import java.text.ParseException; /** * This handles station datasets in "Unidata Observation Dataset v1.0" implemented as linked or contiguous lists. * For "multidimensional Structures" use UnidataStationObsMultidimDataset * * @deprecated use ucar.nc2.ft.point * @author caron */ public class UnidataStationObsDataset extends StationObsDatasetImpl implements TypedDatasetFactoryIF { private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(UnidataStationObsDataset.class); public static boolean isValidFile(NetcdfFile ds) { if (!ds.findAttValueIgnoreCase(null, "cdm_data_type", "").equalsIgnoreCase(FeatureType.STATION.toString()) && !ds.findAttValueIgnoreCase(null, "cdm_datatype", "").equalsIgnoreCase(FeatureType.STATION.toString())) return false; String conv = ds.findAttValueIgnoreCase(null, "Conventions", null); if (conv == null) return false; boolean convOk = false; StringTokenizer stoke = new StringTokenizer(conv, ","); while (stoke.hasMoreTokens()) { String toke = stoke.nextToken().trim(); if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0")) convOk = true; } if (!convOk) return false; // must have this field to be a linked list Variable stationIndexVar = UnidataObsDatasetHelper.findVariable(ds, "parent_index"); if (stationIndexVar == null) return false; return true; } // TypedDatasetFactoryIF public boolean isMine(NetcdfDataset ds) { return isValidFile(ds); } public TypedDataset open(NetcdfDataset ncd, ucar.nc2.util.CancelTask task, StringBuilder errlog) throws IOException { return new UnidataStationObsDataset(ncd); } public UnidataStationObsDataset() {} /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// private Variable latVar, lonVar, altVar, timeVar, timeNominalVar; private Variable lastVar, prevVar, firstVar, nextVar, numChildrenVar; private Variable stationIndexVar, stationIdVar, stationDescVar, numStationsVar; private boolean isForwardLinkedList, isBackwardLinkedList, isContiguousList; private Structure recordVar; private RecordDatasetHelper recordHelper; public UnidataStationObsDataset(NetcdfDataset ds) throws IOException { super(ds); // coordinate variables latVar = UnidataObsDatasetHelper.getCoordinate(ds, AxisType.Lat); lonVar = UnidataObsDatasetHelper.getCoordinate(ds, AxisType.Lon); altVar = UnidataObsDatasetHelper.getCoordinate(ds, AxisType.Height); timeVar = UnidataObsDatasetHelper.getCoordinate(ds, AxisType.Time); if (latVar == null) throw new IllegalStateException("Missing latitude variable"); if (lonVar == null) throw new IllegalStateException("Missing longitude coordinate variable"); if (timeVar == null) throw new IllegalStateException("Missing time coordinate variable"); // variables that link the structures together timeNominalVar = UnidataObsDatasetHelper.findVariable(ds, "time_nominal"); lastVar = UnidataObsDatasetHelper.findVariable(ds, "lastChild"); prevVar = UnidataObsDatasetHelper.findVariable(ds, "prevChild"); firstVar = UnidataObsDatasetHelper.findVariable(ds, "firstChild"); nextVar = UnidataObsDatasetHelper.findVariable(ds, "nextChild"); numChildrenVar = UnidataObsDatasetHelper.findVariable(ds, "numChildren"); stationIndexVar = UnidataObsDatasetHelper.findVariable(ds, "parent_index"); isForwardLinkedList = (firstVar != null) && (nextVar != null); isBackwardLinkedList = (lastVar != null) && (prevVar != null); isContiguousList = !isForwardLinkedList && !isBackwardLinkedList && (firstVar != null) && (numChildrenVar != null); if (!isForwardLinkedList && !isBackwardLinkedList && !isContiguousList) { if (firstVar != null) throw new IllegalStateException("Missing nextVar (linked list) or numChildrenVar (contiguous list) variable"); if (lastVar != null) throw new IllegalStateException("Missing prevVar (linked list) variable"); if (nextVar != null) throw new IllegalStateException("Missing firstVar (linked list) variable"); if (prevVar != null) throw new IllegalStateException("Missing lastVar (linked list) variable"); } // station variables stationIdVar = UnidataObsDatasetHelper.findVariable(ds, "station_id"); stationDescVar = UnidataObsDatasetHelper.findVariable(ds, "station_description"); numStationsVar = UnidataObsDatasetHelper.findVariable(ds, "number_stations"); if (stationIdVar == null) throw new IllegalStateException("Missing station id variable"); // fire up the record helper - LOOK assumes its the record dimension recordHelper = new RecordDatasetHelper(ds, timeVar.getShortName(), timeNominalVar == null ? null : timeNominalVar.getShortName(), dataVariables, parseInfo); recordHelper.setStationInfo(stationIndexVar.getShortName(), stationDescVar == null ? null : stationDescVar.getShortName()); removeDataVariable(stationIndexVar.getShortName()); removeDataVariable(timeVar.getShortName()); if (timeNominalVar != null) removeDataVariable(timeNominalVar.getShortName()); if (prevVar != null) removeDataVariable(prevVar.getShortName()); if (nextVar != null) removeDataVariable(nextVar.getShortName()); recordVar = recordHelper.recordVar; timeUnit = recordHelper.timeUnit; readStations(); // LOOK try to defer this // get min, max date startDate = UnidataObsDatasetHelper.getStartDate(ds); endDate = UnidataObsDatasetHelper.getEndDate(ds); boundingBox = UnidataObsDatasetHelper.getBoundingBox(ds); if (null == boundingBox) setBoundingBox(); // kludge Robb not following spec if (null == startDate) { Variable minTimeVar = ds.findVariable("minimum_time_observation"); int minTimeValue = minTimeVar.readScalarInt(); startDate = timeUnit.makeDate(minTimeValue); } if (null == endDate) { Variable maxTimeVar = ds.findVariable("maximum_time_observation"); int maxTimeValue = maxTimeVar.readScalarInt(); endDate = timeUnit.makeDate(maxTimeValue); } title = ds.findAttValueIgnoreCase(null, "title", ""); desc = ds.findAttValueIgnoreCase(null, "description", ""); } private void readStations() throws IOException { // get the station info Array stationIdArray = stationIdVar.read(); ArrayChar stationDescArray = null; if (stationDescVar != null) stationDescArray = (ArrayChar) stationDescVar.read(); Array latArray = latVar.read(); Array lonArray = lonVar.read(); Array elevArray = (altVar != null) ? altVar.read() : null; Array firstRecordArray = (isForwardLinkedList || isContiguousList) ? firstVar.read() : lastVar.read(); Array numChildrenArray = null; if (numChildrenVar != null) numChildrenArray = numChildrenVar.read(); // how many are valid stations ? Dimension stationDim = UnidataObsDatasetHelper.findDimension(netcdfDataset, "station"); int n = 0; if (numStationsVar != null) n = numStationsVar.readScalarInt(); else n = stationDim.getLength(); // loop over stations Index ima = latArray.getIndex(); recordHelper.stnHash = new HashMap<Object, ucar.unidata.geoloc.Station>(2 * n); for (int i = 0; i < n; i++) { ima.set(i); String stationName; String stationDesc; if (stationIdArray instanceof ArrayChar) { stationName = ((ArrayChar) stationIdArray).getString(i).trim(); stationDesc = (stationDescVar != null) ? stationDescArray.getString(i) : null; if (stationDesc != null) stationDesc = stationDesc.trim(); } else { stationName = stationIdArray.getObject(ima).toString(); stationDesc = (stationDescVar != null) ? (String) stationDescArray.getObject(ima) : null; } UnidataStationImpl bean = new UnidataStationImpl(stationName, stationDesc, latArray.getFloat(ima), lonArray.getFloat(ima), (altVar != null) ? elevArray.getFloat(ima) : Double.NaN, firstRecordArray.getInt(ima), (numChildrenVar != null) ? numChildrenArray.getInt(ima) : -1); stations.add(bean); recordHelper.stnHash.put(i, bean); } } protected void setTimeUnits() {} protected void setStartDate() {} protected void setEndDate() {} protected void setBoundingBox() { boundingBox = stationHelper.getBoundingBox(); } public List getData(CancelTask cancel) throws IOException { List<StationObsDatatype> allData = new ArrayList<StationObsDatatype>(); int n = getDataCount(); for (int i = 0; i < n; i++) { StationObsDatatype obs = makeObs(i, false, null); if (obs != null) allData.add(obs); if ((cancel != null) && cancel.isCancel()) return null; } return allData; } public int getDataCount() { Dimension unlimitedDim = netcdfDataset.getUnlimitedDimension(); return unlimitedDim.getLength(); } public List getData(ucar.unidata.geoloc.Station s, CancelTask cancel) throws IOException { return ((UnidataStationImpl) s).getObservations(); } public void checkLinks(StringBuffer sbuff) throws IOException { int countErrs = 0; if (isBackwardLinkedList) { Array lastArray = lastVar.read(); Array prevArray = prevVar.read(); Array stnArray = stationIndexVar.read(); Index prevIndex = prevArray.getIndex(); Index stnIndex = stnArray.getIndex(); int stnIdx = 0; IndexIterator stnIter = lastArray.getIndexIterator(); while (stnIter.hasNext()) { Set<Integer> records = new HashSet<Integer>(500); // for each station, follow the links int recNo = stnIter.getIntNext(); System.out.print("Station " + stnIdx); while (recNo >= 0) { System.out.print(" " + recNo); records.add(recNo); int stnFromRecord = stnArray.getInt(stnIndex.set(recNo)); if (stnFromRecord != stnIdx) { sbuff.append("recno ").append(recNo).append(" has bad station index\n"); countErrs++; if (countErrs > 10) return; } // get next one recNo = prevArray.getInt(prevIndex.set(recNo)); if (records.contains(recNo)) { sbuff.append("stn ").append(stnIdx).append(" has circular links\n"); countErrs++; if (countErrs > 10) return; break; } } System.out.println(); stnIdx++; } } sbuff.append("done"); } //////////////////////////////////////////////////////// private class UnidataStationImpl extends StationImpl { private int firstRecord; private Variable next; private UnidataStationImpl(String name, String desc, double lat, double lon, double elev, int firstRecord, int count) { super(name, desc, lat, lon, elev, count); this.firstRecord = firstRecord; next = (isForwardLinkedList) ? nextVar : prevVar; } protected List<StationObsDatatype> readObservations() throws IOException { List<StationObsDatatype> obs = new ArrayList<StationObsDatatype>(); int recno = firstRecord; int end = firstRecord + count - 1; int nextRecord = firstRecord; while (recno >= 0) { try { // deal with files that are updating if (recno > getDataCount()) { int n = getDataCount(); netcdfDataset.syncExtend(); // LOOK kludge? log.info("UnidataStationObsDataset.makeObs recno=" + recno + " > " + n + "; after sync= " + getDataCount()); } StructureData sdata = recordVar.readStructure(recno); if (isContiguousList) { if (nextRecord++ > end) break; } else { nextRecord = sdata.getScalarInt(next.getShortName()); } double obsTime = getTime(timeVar, sdata); double nomTime = (timeNominalVar == null) ? obsTime : getTime(timeNominalVar, sdata); obs.add(recordHelper.new RecordStationObs(this, obsTime, nomTime, recno)); recno = nextRecord; } catch (ucar.ma2.InvalidRangeException e) { log.error("UnidataStationObsDataset.readObservation recno=" + recno, e); throw new IOException(e.getMessage()); } catch (ParseException e) { log.error("UnidataStationObsDataset.readObservation recno=" + recno, e); throw new IOException(e.getMessage()); } } Collections.sort(obs); return obs; } DataIterator iterator() { return new StationIterator(); } DataIterator iterator(Date start, Date end) { return new StationIterator(start, end); } private class StationIterator implements DataIterator { int nextRecno = firstRecord; int last = firstRecord + count - 1; // contiguous only double startTime, endTime; boolean hasDateRange; StationIterator() {} StationIterator(Date start, Date end) { startTime = timeUnit.makeValue(start); endTime = timeUnit.makeValue(end); hasDateRange = true; } public boolean hasNext() { return (nextRecno >= 0); } public Object nextData() throws IOException { RecordDatasetHelper.RecordStationObs sobs = makeObs(nextRecno, true, null); if (!sobs.getStation().getName().equals(getName())) throw new IllegalStateException("BAD Station link (" + nextRecno + ") station name=" + sobs.getStation().getName() + " should be " + getName()); if (isContiguousList) { nextRecno++; if (nextRecno > last) nextRecno = -1; } else { nextRecno = sobs.sdata.getScalarInt(next.getShortName()); } if (hasDateRange) { double timeValue = sobs.getObservationTime(); if ((timeValue < startTime) || (timeValue > endTime)) return nextData(); } return sobs; } public Object next() { try { return nextData(); } catch (IOException e) { log.error("UnidataStationObsDataset.StationIterator.next recno=" + nextRecno, e); throw new IllegalStateException(e.getMessage()); // not really an illegal state... } } public void remove() { throw new UnsupportedOperationException(); } } } protected RecordDatasetHelper.RecordStationObs makeObs(int recno, boolean storeData, StructureData sdata) throws IOException { try { // deal with files that are updating if (recno > getDataCount()) { int n = getDataCount(); netcdfDataset.syncExtend(); // LOOK kludge? log.info("UnidataStationObsDataset.makeObs recno=" + recno + " > " + n + "; after sync= " + getDataCount()); } if (null == sdata) sdata = recordVar.readStructure(recno); // find the station int stationIndex = sdata.getScalarInt(stationIndexVar.getShortName()); if (stationIndex < 0 || stationIndex >= stations.size()) { parseInfo.append("cant find station at index = ").append(stationIndex).append("\n"); return null; } ucar.unidata.geoloc.Station station = (ucar.unidata.geoloc.Station) stations.get(stationIndex); if (station == null) { parseInfo.append("cant find station at index = ").append(stationIndex).append("\n"); return null; } double obsTime = getTime(timeVar, sdata); double nomTime = (timeNominalVar == null) ? obsTime : getTime(timeNominalVar, sdata); if (storeData) return recordHelper.new RecordStationObs(station, obsTime, nomTime, sdata); else return recordHelper.new RecordStationObs(station, obsTime, nomTime, recno); } catch (ucar.ma2.InvalidRangeException e) { log.error("UnidataStationObsDataset.makeObs recno=" + recno, e); throw new IOException(e.getMessage()); } catch (ParseException e) { log.error("UnidataStationObsDataset.makeObs recno=" + recno, e); throw new IOException(e.getMessage()); } } public DataIterator getDataIterator(ucar.unidata.geoloc.Station s) { return ((UnidataStationImpl) s).iterator(); } public DataIterator getDataIterator(ucar.unidata.geoloc.Station s, Date start, Date end) { return ((UnidataStationImpl) s).iterator(start, end); } public DataIterator getDataIterator(int bufferSize) throws IOException { return new StationDatatypeIterator(recordHelper.recordVar, bufferSize); } private class StationDatatypeIterator extends DatatypeIterator { protected Object makeDatatypeWithData(int recnum, StructureData sdata) throws IOException { return makeObs(recnum, true, sdata); } StationDatatypeIterator(Structure struct, int bufferSize) { super(struct, bufferSize); } } }