package nl.wldelft.fews.system.plugin.dataImport;
import hdf.hdf5lib.H5;
import hdf.hdflib.HDFLibrary;
import hdf.object.Attribute;
import hdf.object.FileFormat;
import hdf.object.Group;
import hdf.object.HObject;
import hdf.object.ScalarDS;
import hdf.object.h5.H5File;
import nl.wldelft.util.DateUtils;
import nl.wldelft.util.FastDateFormat;
import nl.wldelft.util.Interruption;
import nl.wldelft.util.SystemUtils;
import nl.wldelft.util.TimeZeroConsumer;
import nl.wldelft.util.coverage.Geometry;
import nl.wldelft.util.coverage.GridUtils;
import nl.wldelft.util.coverage.RegularGridGeometry;
import nl.wldelft.util.geodatum.GeoDatum;
import nl.wldelft.util.geodatum.GeoPoint;
import nl.wldelft.util.geodatum.Wgs1984Point;
import nl.wldelft.util.io.FileParser;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.LockableContentHandler;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import javax.swing.tree.DefaultMutableTreeNode;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.Date;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
//ToDo preferably refactor the HDF5 readers (KnmiHdf5, LandsathHf5, DxOnline) since they are all dedicated due to hard coded
//ToDo parameter names and dateTime field names. See info in the issues FEWS-FEWS-12362 and FEWS-8037
public final class KnmiHdf5Parser implements FileParser<TimeSeriesContentHandler>, TimeZeroConsumer {
private static final Logger log = LogManager.getLogger();
static {
System.setProperty(HDFLibrary.HDFPATH_PROPERTY_KEY, loadHdfLibrary());
System.setProperty(H5.H5PATH_PROPERTY_KEY, loadHdf5Library());
}
private static String loadHdf5Library() {
return SystemUtils.loadLibrary(KnmiHdf5Parser.class.getClassLoader(), "jhdf5");
}
private static String loadHdfLibrary() {
return SystemUtils.loadLibrary(KnmiHdf5Parser.class.getClassLoader(), "jhdf");
}
public static final String readerType = "KNMI-HDF5";
private File file = null;
private TimeSeriesContentHandler contentHandler = null;
private String gridStartPoint = "NW";
private long timeZero = DateUtils.roundTimeToWholeSeconds(System.currentTimeMillis());
private boolean readObservationTimes = true; //will be set to false if <fileNameObservationDateTimePattern> is configured in TimeSeriesImportRun.xml
private H5File h5FileReader = null;
/* List of known groups - actually pairs of names:
The first name is the group directly under the root
The second name is the subgroup of the first, if it exists,
otherwise an empty string
For each group we also need the missing value
KNMI radar files store the data as unsigned two-bytes integers
*/
private final String[] knownParameter = new String[]{
"image1", "image_data",
"image", "data",
"image", "Neerslag",
"dataset_DXk", "image",
"dataset1/data1","data",
"Ourthe/Average", "CorrectedApriori",
"Ourthe/Average", "CorrectedAprioriCatchmentAverage",
"Ourthe/Average", "CorrectedPosterori",
"Ourthe/Average", "CorrectedPosterioriCatchmentAverage",
"Ourthe/Average", "Uncorrected",
"Ourthe/Average", "UncorrectedCatchmentAverage",
"Ourthe/Percentile10", "CorrectedApriori",
"Ourthe/Percentile10", "CorrectedAprioriCatchmentAverage",
"Ourthe/Percentile25", "CorrectedApriori",
"Ourthe/Percentile25", "CorrectedAprioriCatchmentAverage",
"Ourthe/Percentile50", "CorrectedApriori",
"Ourthe/Percentile50", "CorrectedAprioriCatchmentAverage",
"Ourthe/Percentile75", "CorrectedApriori",
"Ourthe/Percentile75", "CorrectedAprioriCatchmentAverage",
"Ourthe/Percentile90", "CorrectedApriori",
"Ourthe/Percentile90", "CorrectedAprioriCatchmentAverage",
"Totalscan/Average", "CorrectedApriori",
"Totalscan/Average", "CorrectedAprioriCatchmentAverage",
"Totalscan/Average", "CorrectedPosterori",
"Totalscan/Average", "CorrectedPosterioriCatchmentAverage",
"Totalscan/Average", "Uncorrected",
"Totalscan/Average", "UncorrectedCatchmentAverage",
"Totalscan/Percentile10", "CorrectedApriori",
"Totalscan/Percentile10", "CorrectedAprioriCatchmentAverage",
"Totalscan/Percentile25", "CorrectedApriori",
"Totalscan/Percentile25", "CorrectedAprioriCatchmentAverage",
"Totalscan/Percentile50", "CorrectedApriori",
"Totalscan/Percentile50", "CorrectedAprioriCatchmentAverage",
"Totalscan/Percentile75", "CorrectedApriori",
"Totalscan/Percentile75", "CorrectedAprioriCatchmentAverage",
"Totalscan/Percentile90", "CorrectedApriori",
"Totalscan/Percentile90", "CorrectedAprioriCatchmentAverage",
//Global Precipitation Mission files (FEWS-12362), file example : 3B-HHR.MS.MRG.3IMERG.20140708-S223000-E225959.1350.V03D.HDF5
"Grid","HQprecipitation",
"Grid","IRprecipitation",
"Grid","precipitationCal",
"Grid","precipitationUncal",
//Global Precipitation Mission files (FEWS-12362), file example : 2B.GPM.DPRGMI.CORRA2014.20140731-S172035-E185308.002397.V03C.HDF5
"MS","airPressure",
"MS","airTemperature",
"MS","cloudIceWaterCont",
"MS","cloudLiqWaterCont",
"MS","correctedReflectFactor",
"MS","liqMassFracTrans",
"MS","liqRateFracTrans",
"MS","pia",
"MS","precipTotPSDparamHigh",
"MS","precipTotPSDparamLow",
"MS","precipTotRate",
"MS","precipTotRateSigma",
"MS","precipTotWaterCont",
"MS","precipTotWaterContSigma",
"MS","simulatedBrightTemp",
"MS","skinTemperature",
"MS","surfEmissivity",
"MS","surfLiqRateFrac",
"MS","surfPrecipTotRate",
"MS","surfPrecipTotRateSigma",
"MS","surfaceAirPressure",
"MS","surfaceAirTemperature",
"MS","surfaceVaporDensity",
"MS","tenMeterWindSpeed",
"MS","vaporDensity",
//Global Precipitation Mission files (FEWS-12362), file example : 1C.F16.SSMIS.XCAL2014-V.20141231-S013950-E032144.057795.V02B.HDF5
"S1","incidenceAngle",
"S1","sunGlintAngle",
"S1","Quality",
//Global Precipitation Mission files (FEWS-12362), file example : 2A.GPM.Ka.V5-20140829.20140731-S141525-E154759.002395.V03B.HDF5
//ToDo this file has much more scalarDS objects, the user should specify which objects are parameter objects. Presently only 5 'parameters' are included
"MS/PRE","localZenithAngle",
"MS/PRE","localZenithAngle",
"MS/PRE","binRealSurface",
"MS/PRE","binStormTop",
"MS/PRE","heightStormTop",
//Global Precipitation Mission files (FEWS-12362), file example : 3A-DAY.NOAA18.MHS.GRID2014R2.20140731-S000000-E235959.212.V03B.HDF5
"Grid","cloudWater",
"Grid","cloudWaterPath",
"Grid","convectPrecipFraction",
"Grid","fractionQuality0",
"Grid","fractionQuality1",
"Grid","fractionQuality2",
"Grid","iceWater",
"Grid","iceWaterPath",
"Grid","latentHeat",
"Grid","liquidPrecipFraction",
"Grid","mixedWater",
"Grid","mixedWaterPath",
"Grid","rainWater",
"Grid","rainWaterPath",
"Grid","surfacePrecipitation",
// SMAP_L3_SM_P_E_20150331_R16510_001.h5 Albrecht
"Soil_Moisture_Retrieval_Data_AM","soil_moisture",
"Soil_Moisture_Retrieval_Data_PM","soil_moisture_pm"
};
/**
* Constructs KnmiHdf5Parser
* @param configFilename name of configuration file. Is allowed to be NULL
*/
KnmiHdf5Parser(String gridStartingPoint) {
if (gridStartPoint == null) {
throw new IllegalArgumentException("gridStartPoint == null");
}
if (gridStartPoint.equalsIgnoreCase("NW") && gridStartPoint.equalsIgnoreCase("SW")) {
throw new IllegalArgumentException("gridStartPoint must be NW or SW. Only flipping from SW supported !");
}
this.gridStartPoint = gridStartingPoint;
}
@Override
public void setTimeZero(long timeZero) {
this.timeZero = timeZero;
}
@Override
public void parse(File file, TimeSeriesContentHandler contentHandler) throws Exception {
this.file = file;
if (contentHandler instanceof LockableContentHandler) this.readObservationTimes = false;
// Request the implementing class of FileFormat: H5File
FileFormat h5file = FileFormat.getFileFormat(FileFormat.FILE_TYPE_HDF5);
if (h5file == null) {
Throwable e = null;
try {
Class clz = Class.forName("hdf.object.h5.H5File");
clz.newInstance();
} catch (Interruption | ThreadDeath ex) {
throw ex;
} catch (Throwable err) {
e = err;
}
throw new Exception("Could not load HDF5 library - probable cause: MicroSoft VC90 libraries not found: " + e, e);
}
// Create an instance of H5File object with read/write access
h5FileReader = (H5File)h5file.open(file.getAbsolutePath(), FileFormat.READ);
// H5File h5file = new H5File();
// h5FileReader = (H5File) h5file.createFile(this.filename,FileFormat.READ);
// Open the file and load the file structure; file id is returned.
int fid = h5FileReader.open();
this.contentHandler = contentHandler;
try {
read();
} finally {
if (h5FileReader != null) h5FileReader.close();
}
}
private void read() throws Exception {
int fileID = h5FileReader.open();
int[] parameterIds = readParameters();
//Loop over the available parameters
for (int i = 0; i <parameterIds.length; i ++) {
DefaultTimeSeriesHeader tmsHeader = new DefaultTimeSeriesHeader();
String parameterName = getParameterName(parameterIds[i]);
tmsHeader.setParameterId(parameterName);
this.contentHandler.setNewTimeSeriesHeader(tmsHeader);
if (!this.contentHandler.isCurrentTimeSeriesHeaderForAllTimesRejected()) {
if (log.isDebugEnabled()) log.debug("Reading parameter " + parameterName);
long[] times;
if (this.readObservationTimes) {
times = readTimes(parameterIds[i]); //Returns always 1 time !
} else {
times = new long[]{System.currentTimeMillis()};
}
//Get grid geometry from the Hdf5 grid geometry:
Geometry geometry = getGeometry(parameterIds[i]);
//Review remarks: no clue why this loop over the times is necessary !
//Loop over the timesteps
for (int t = 0; t < times.length; t++) {
int idx = parameterIds[i];
ScalarDS image = (ScalarDS) findObject(h5FileReader, "/" + knownParameter[2 * idx] + "/" + knownParameter[2 * idx + 1] + "/");
float[] values = Hdf5TimeSeriesParserUtils.readImageData(image);
if (gridStartPoint.equalsIgnoreCase("SW")) {
//Reflect the grid in the middle row
GridUtils.reverseOrderRows(values, geometry);
}
this.contentHandler.setTime(times[t]);
this.contentHandler.setGeometry(geometry);
this.contentHandler.setCoverageValues(values);
this.contentHandler.applyCurrentFields();
}
}
}
}
private int[] readParameters() throws IOException {
int count = 0;
int[] idx = new int[knownParameter.length / 2];
for (int i = 0; i < knownParameter.length; i += 2) {
if (parameterExists(knownParameter[i], knownParameter[i+1])) {
// Very simple for the moment!
idx[count] = i / 2;
count++;
}
}
int[] knownIdx = new int[count];
for (int i =0; i < count; i++) {
knownIdx[i] = idx[i];
}
return knownIdx;
}
private boolean parameterExists(String group, String element) {
ScalarDS parameter = (ScalarDS) findObject(h5FileReader, "/" + group + "/" + element + "/");
return parameter != null;
}
private String getParameterName(int idx) {
String name;
// Very simple for the moment!
if (knownParameter[2 * idx + 1].equals("")) {
name = knownParameter[2 * idx];
} else {
if (idx != 0) {
name = knownParameter[2 * idx] + '/' + knownParameter[2 * idx + 1];
} else {
name = knownParameter[2 * idx + 1]; // For compatibility
}
}
return name;
}
// AM: copied from documentation
static HObject findObject(FileFormat file, String path)
{
boolean found = false;
if (file == null || path == null)
return null;
if (!path.endsWith("/"))
path = path+"/";
DefaultMutableTreeNode theRoot =
(DefaultMutableTreeNode)file.getRootNode();
if (theRoot == null)
return null;
else if (path.equals("/"))
return (HObject)theRoot.getUserObject();
Enumeration local_enum =
((DefaultMutableTreeNode)theRoot).breadthFirstEnumeration();
DefaultMutableTreeNode theNode = null;
HObject theObj = null;
while(local_enum.hasMoreElements())
{
theNode = (DefaultMutableTreeNode)local_enum.nextElement();
theObj = (HObject)theNode.getUserObject();
String fullPath = theObj.getFullName()+"/";
if (path.equals(fullPath)) {
found = true;
break;
}
}
return (found ? theObj : null);
}
private long[] readTimes(int idx) throws IOException {
// Variable with date/time are hard coded. Presently these variable names are supported : "/overview/", "/dataset1/what/"
long[] times = extractTimesFromFile();
if (times == null) times = extractTimeFromVariableDataset1What();
if ( times == null ) {
// Quick and dirty: get the date/time from the file name
try {
int posdot = file.getName().lastIndexOf(".");
var dateFormat = FastDateFormat.getInstance("yyyyMMddHHmm", contentHandler.getDefaultTimeZone(), Locale.US, null);
long reftime = dateFormat.parse(file.getName().substring(posdot - 12,posdot)).getTime();
times = new long[1];
times[0] = reftime ;
} catch (Exception e) {
return new long[]{this.timeZero}; // Keep parsing time from file name for backward compatibility, but do not log any error . Use fileNameObservationDateTimePattern in the import module
}
if (log.isDebugEnabled()) log.debug("Retrieving one time - " + new Date(times[0]));
}
return times;
}
private long[] extractTimesFromFile() {
long[] times = new long[1];
String dateTimeString = "";
var dateFormat = FastDateFormat.getInstance("dd-MM-yyyy;HH:mm:ss:SSS", contentHandler.getDefaultTimeZone(), Locale.US, null);
Group dateTime = (Group) findObject(h5FileReader, "/overview/");
int error = 1;
if (dateTime != null) {
error = 0;
try {
List attributes = dateTime.getMetadata();
for (Iterator iterator = attributes.iterator(); iterator.hasNext();) {
Attribute attribute = (Attribute) iterator.next();
if ( attribute.getName().equals("product_datetime_end")) {
dateTimeString = dateTimeString + ((String[]) attribute.getValue())[0];
}
}
dateTimeString = correctedDate(dateTimeString);
} catch (Exception e) {
error = 1;
}
}
if ( error != 0 ) {
return null;
} else {
try {
times[0] = dateFormat.parse(dateTimeString).getTime();
} catch (ParseException e) {
log.error("Error parsing date/time %s (from %s): %s", dateTimeString, file, e);
return null;
}
}
return times;
}
/**
* Read date and time from variable /dataset1/what/
*
* @return date/time in millis . Returns Long.MIN_VALUE if variable "/dataset1/what/" cannot be found, or
* if "enddate" and/or "endtime" attribute cannot be found or if any error occurs while reading
*/
private long[] extractTimeFromVariableDataset1What() {
Group dateTime = (Group) findObject(h5FileReader, "/dataset1/what/");
if (dateTime == null) return null;
String dateString = "";
String timeString = "";
try {
List attributes = dateTime.getMetadata();
for (Iterator iterator = attributes.iterator(); iterator.hasNext(); ) {
Attribute attribute = (Attribute) iterator.next();
if (attribute.getName().equals("enddate")) dateString = ((String[]) attribute.getValue())[0];
if (attribute.getName().equals("endtime")) timeString = ((String[]) attribute.getValue())[0];
}
} catch (Exception e) {
return null;
}
// Variable "/dataset1/what/" should have attribute 'enddate' and 'endtime', for example :enddate = "20140417"; :endtime = "210106";
var dateFormat = FastDateFormat.getInstance("yyyyMMddHHmmss", contentHandler.getDefaultTimeZone(), Locale.US, null);
long res;
try {
res = dateFormat.parse(dateString + timeString).getTime();
} catch (ParseException e) {
log.error("Error parsing date/time %s%s (from %s): %s", dateString, timeString, file, e);
return null;
}
return new long[]{res};
}
private static String correctedDate(String date) {
// Adjust the date string - DateFormat will not accept the month name in capitals
String[] monthName = new String[]{"JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "OKT", "NOV", "DEC"};
String[] monthNumber = new String[]{"01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "10", "11", "12"};
String month = date.substring(3,6);
int m = 0;
for (int i = 0; i <monthName.length; i++) {
if ( month.equalsIgnoreCase(monthName[i])) {
m = i;
break;
}
}
// Also take care of the variation in the separator for the seconds and miilseconds - sometimes : and sometimes .
return date.substring(0,3) + monthNumber[m] + date.substring(6,20) + ':' + date.substring(21);
}
private Geometry getGeometry(int idx) throws Exception {
ScalarDS parameter = (ScalarDS) findObject(h5FileReader, "/" + knownParameter[2*idx] + "/" + knownParameter[2*idx+1] + "/");
// Force the library to read the data, only then do we get information on the dimensions.
// (Note: it might be more efficient to store the parameter and its data for later use. However,
// we may want to retrieve the data per timestep and it is likely to be a micro-optimisation
// anyway)
assert parameter != null;
var parameterData = parameter.getData(); // load data
assert parameterData != null;
long[] shape = new long[2];
shape[0] = Math.max(parameter.getHeight(), 1);
shape[1] = Math.max(parameter.getWidth(), 1);
// These parameters are completely arbitrary: most HDf5 files read by this parser do not publish the grid clearly!
double latFirst = 0.0;
double lonFirst = 0.0;
double dlat = 50.0/shape[1];
double dlon = 50.0/shape[0];
GeoPoint first = new Wgs1984Point(latFirst, lonFirst);
return RegularGridGeometry.create(GeoDatum.WGS_1984, first, dlon, dlat,
(int)shape[0], (int)shape[1]);
}
}
Overview
Content Tools
