NetCDF file reader for LocalFileInterface

Benjamin Renard
1 parent 03e8f845
Showing 9 changed files with 390 additions and 11 deletions Show diff stats
config/xsd/localbase/base.xsd
src/ParamGetImpl/LocalFileInterface/FileReaderNetCDF.cc
src/ParamGetImpl/LocalFileInterface/FileReaderNetCDF.hh
src/ParamGetImpl/LocalFileInterface/VirtualInstrumentBaseParser.cc
src/Parameters/ParameterManager.cc
src/Parameters/ParameterManager.hh
test/data/DataBaseParameters/local_netcdf_b.xml
test/data/DataBaseParameters/local_netcdf_v.xml
test/data/LocalBase/base.xml
@@ -21,6 +21,7 @@
               <xs:enumeration value="ASCII" />
               <xs:enumeration value="CDF" />
               <xs:enumeration value="VOT" />
+              <xs:enumeration value="NC" />
             </xs:restriction>
           </xs:simpleType>
     	</xs:attribute>
@@ -33,4 +34,4 @@
     	<xs:attribute name="start" type="xs:string" use="required" />
     	<xs:attribute name="stop" type="xs:string" use="required" />
     </xs:complexType>
-</xs:schema>
 \ No newline at end of file
+</xs:schema>
@@ -18,6 +18,7 @@ namespace LocalFileInterface {
  
 FileReaderNetCDF::FileReaderNetCDF()
 {
+	resetFile();
 }
  
 FileReaderNetCDF::~FileReaderNetCDF()
@@ -26,38 +27,170 @@ FileReaderNetCDF::~FileReaderNetCDF()
  
 bool FileReaderNetCDF::open(std::string filePath)
 {
-	return false;
+	boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib);
+	if (isOpened()) {
+		return true;
+	}
+
+	int status = nc_open(filePath.c_str(), 0, &_file.id);
+	if (status != NC_NOERR) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::open - Cannot open netCDF file : " << getErrorMessage(status))
+		resetFile();
+		return false;
+	}
+	return loadNetCDFFileInfo();
 }
  
 bool FileReaderNetCDF::close(void)
 {
-	return false;
+	boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib);
+	if (!isOpened()) {
+		return true;
+	}
+	int status = nc_close(_file.id);
+	if (status != NC_NOERR) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::close - Cannot close netCDF file : " << getErrorMessage(status))
+		return false;
+	}
+	resetFile();
+	return true;
 }
  
 bool FileReaderNetCDF::isOpened(void)
 {
-	return false;
+	return (_file.id != 0);
 }
  
 std::string FileReaderNetCDF::getTimeParamId(void)
 {
+	if (!isOpened()) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getTimeParamId - File not opened")
+		return "";
+	}
+
+	for (std::vector<NetCDFVar>::iterator it =  _file.vars.begin(); it != _file.vars.end(); ++it) {
+		std::string name = it->name;
+		std::transform(name.begin(), name.end(), name.begin(), ::tolower);
+		if ((std::strcmp(name.c_str(),"time") == 0) && (it->type == NC_DOUBLE)) {
+			return it->name;
+		}
+	}
+
+	LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getTimeParamId - Cannot find the time variable")
 	return "";
 }
  
 bool FileReaderNetCDF::getParamInfo(std::string& paramId, LocalParamType& paramType, int& dim1Size, int& dim2Size)
 {
-	return false;
+	for (std::vector<NetCDFVar>::iterator it =  _file.vars.begin(); it != _file.vars.end(); ++it) {
+		if (std::strcmp(it->name, paramId.c_str()) == 0) {
+			if (it->dimids.size() == 1) {
+				//scalar
+				dim1Size = 1;
+				dim2Size = 1;
+			}
+			else if (it->dimids.size() == 2) {
+				//vector
+				dim1Size = _file.dims[it->dimids[1]].length;
+				dim2Size = 1;
+			}
+			else if (it->dimids.size() == 3) {
+				//Tab2D
+				dim1Size = _file.dims[it->dimids[1]].length;
+				dim2Size = _file.dims[it->dimids[2]].length;
+			}
+			else {
+				LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamInfo - Variable dimensions not supported")
+				return false;
+			}
+			switch (it->type) {
+				case NC_FLOAT :
+					paramType = LocalParamType::TYPE_FLOAT;
+					break;
+				case NC_DOUBLE :
+					paramType = LocalParamType::TYPE_DOUBLE;
+					break;
+				case NC_SHORT :
+					paramType = LocalParamType::TYPE_SHORT;
+					break;
+				case NC_INT :
+					paramType = LocalParamType::TYPE_INT;
+					break;
+				default:
+					paramType = LocalParamType::TYPE_UNKNOWN;
+					LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamInfo - Unknown variable type")
+					return false;
+			}
+		}
+	}
+	return true;
 }
  
 int FileReaderNetCDF::getRecordIndex(std::string& timeId, double time)
 {
+	boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib);
+	for (std::vector<NetCDFVar>::iterator it =  _file.vars.begin(); it != _file.vars.end(); ++it) {
+		if (std::strcmp(it->name, timeId.c_str()) == 0) {
+			if (!loadData(*it)) {
+				LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getRecordIndex - Cannot load time data")
+				return -1;
+			}
+			double* buffer = (double*)it->buffer;
+			for (int i = 0; i < getNbVarRecords(*it); ++i) {
+				if (time <= buffer[i]) {
+					return i;
+				}	
+			}
+		}
+	}
 	return -1;
 }
  
 FileReaderStatus FileReaderNetCDF::getParamPacketData(std::string& timeId, std::string& paramId,
 		int recordIndex, double stopTime, LocalParamDataPacket *packet)
 {
-	return FRS_ERROR;
+	boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib);
+
+	NetCDFVar* timeVar = NULL;
+	NetCDFVar* paramVar = NULL;
+	for (std::vector<NetCDFVar>::iterator it =  _file.vars.begin(); it != _file.vars.end(); ++it) {
+		if (std::strcmp(it->name, timeId.c_str()) == 0) {
+			timeVar = &(*it);
+		}
+		if (std::strcmp(it->name, paramId.c_str()) == 0) {
+			paramVar = &(*it);
+		}
+	}
+
+	if ((timeVar == NULL) || (paramVar == NULL)) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Cannot retrieve time or param variable")
+		return FRS_ERROR;
+	}
+
+	if (!loadData(*timeVar) || !loadData(*paramVar)) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Cannot load time or param data")
+		return FRS_ERROR;
+	}
+
+	double* time_buffer = (double*)timeVar->buffer;
+	bool packetFull = false;
+	int nbValues = getNbVarValuesByRecord(*paramVar);
+	for (int i = recordIndex; i < getNbVarRecords(*paramVar); ++i) {
+		double crtTime = time_buffer[i];
+		void* pos = (void*)((char*)paramVar->buffer + paramVar->valuesize * nbValues * i);
+		if (crtTime > stopTime) {
+			return FRS_FINISH;
+		}
+		if (!packet->addData(crtTime, pos, packetFull))
+		{
+			if (packetFull)
+				return FRS_MORE;
+			LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Error to add data in packet");
+			return FRS_ERROR;
+                }
+	}
+
+	return FRS_EOF;
 }
  
 bool FileReaderNetCDF::getInfo(const char* /*pInfoName*/, std::vector<double>& /*res*/)
@@ -65,5 +198,167 @@ bool FileReaderNetCDF::getInfo(const char* /*pInfoName*/, std::vector&lt;double&gt;&amp; /
 	return false;
 }
  
+void FileReaderNetCDF::resetFile() {
+	_file.id = 0;
+	_file.dims.clear();
+	for (std::vector<NetCDFVar>::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) {
+		if (it->buffer != NULL) {
+			std::free(it->buffer);
+			it->buffer = NULL;
+		}
+	}
+	_file.vars.clear();
+	_file.atts.clear();
+}
+
+bool FileReaderNetCDF::loadNetCDFFileInfo() {
+	if (!isOpened()) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - File not opened")
+		return false;
+	}
+
+	int status, ndims, nvars, ngatts, unlimdimid;
+	status = nc_inq(_file.id, &ndims, &nvars, &ngatts, &unlimdimid);
+
+	if (status != NC_NOERR) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load info in netCDF file : " << getErrorMessage(status))
+		return false;
+	}
+
+	//Load dims
+	for (int i = 0; i < ndims; ++i) {
+		NetCDFDim dim;
+		status = nc_inq_dim(_file.id, i, dim.name, &dim.length);
+		if (status != NC_NOERR) {
+			LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load dimension info : " << getErrorMessage(status))
+			return false;
+		}
+		dim.isUnlimited = (i == unlimdimid);
+		_file.dims.push_back(dim);
+	}
+
+	//Load global attributes
+	for (int i = 0; i < ngatts; ++i) {
+		NetCDFAtt att;
+		status = nc_inq_attname(_file.id, NC_GLOBAL, i, att.name);
+		if (status != NC_NOERR) {
+			LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load global attribute name : " << getErrorMessage(status))
+			return false;
+		}
+		status = nc_inq_att(_file.id, NC_GLOBAL, att.name, &att.type, &att.length);
+		if (status != NC_NOERR) {
+			LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load global attribute info : " << getErrorMessage(status))
+			return false;
+		}
+		_file.atts.push_back(att);
+	}
+
+	//Load variables
+	for (int i = 0; i < nvars; ++i) {
+		int natts;
+		int ndims;
+		int dimids[NC_MAX_VAR_DIMS];
+		NetCDFVar var;
+		var.buffer = NULL;
+		status = nc_inq_var(_file.id, i, var.name, &var.type, &ndims, dimids, &natts);
+		if (status != NC_NOERR) {
+			LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable info : " << getErrorMessage(status))
+			return false;
+		}
+		if (ndims > 3) {
+			//Not supported by the kernel
+			continue;
+		}
+		for (int j = 0; j < ndims; ++j) {
+			var.dimids.push_back(dimids[j]);
+		}
+		//Load variable attributes
+		for (int j = 0; j < natts; ++j) {
+			NetCDFAtt att;
+			status = nc_inq_attname(_file.id, i, j, att.name);
+			if (status != NC_NOERR) {
+				LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable attribute : " << getErrorMessage(status))
+				return false;
+			}
+			status = nc_inq_att(_file.id, i, att.name, &att.type, &att.length);
+			if (status != NC_NOERR) {
+				LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable attribute : " << getErrorMessage(status))
+				return false;
+			}
+			var.atts.push_back(att);
+		}
+		_file.vars.push_back(var);
+	}
+	return true;
+}
+
+int FileReaderNetCDF::getNbVarValuesByRecord(const NetCDFVar& var) {
+	if (var.dimids.size() == 1) {
+		//scalar
+		return 1;
+	}
+	else if (var.dimids.size() == 2) {
+		//vector
+		return _file.dims[var.dimids[1]].length;
+	}
+	else if (var.dimids.size() == 3) {
+		//tab2d
+		return _file.dims[var.dimids[1]].length * _file.dims[var.dimids[2]].length;
+	}
+	return 1;
+}
+
+int FileReaderNetCDF::getNbVarRecords(const NetCDFVar& var) {
+	if (var.dimids.size() < 1)
+		return 0;
+	return _file.dims[var.dimids[0]].length;
+}
+
+bool FileReaderNetCDF::loadData(NetCDFVar& var) {
+	if (var.buffer != NULL) {
+		return true;
+	}
+	int nbRecords = getNbVarRecords(var);
+	int nbValues = getNbVarValuesByRecord(var);
+	int varid;
+	int status = nc_inq_varid (_file.id, var.name, &varid);
+	if (status != NC_NOERR) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadData - Cannot load variable data : " << getErrorMessage(status))
+		return false;
+	}
+	switch (var.type) {
+		case NC_FLOAT :
+			var.valuesize = sizeof(float);
+			break;
+		case NC_DOUBLE :
+			var.valuesize = sizeof(double);
+			break;
+		case NC_SHORT :
+			var.valuesize = sizeof(short);
+			break;
+		case NC_INT :
+			var.valuesize = sizeof(int);
+			break;
+		default:
+			return false;
+	}
+	var.buffer = std::malloc(nbRecords*nbValues*var.valuesize);
+	status = nc_get_var(_file.id, varid,  var.buffer);
+	if (status != NC_NOERR) {
+		LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadData - Cannot load variable data : " << getErrorMessage(status))
+		std::free(var.buffer);
+		var.buffer = NULL;
+		return false;
+	}
+	return true;
+}
+
+std::string FileReaderNetCDF::getErrorMessage(int status) {
+	if (status == NC_NOERR) {
+		return "NO ERROR";
+	}
+	return nc_strerror(status);
+}
+
 } /* LocalFileInterface */
 } /* AMDA */
@@ -12,6 +12,8 @@
  
 #include "netcdf.h"
  
+#include <vector>
+
 namespace AMDA {
 namespace LocalFileInterface {
  
@@ -72,6 +74,51 @@ public:
 	 */
 	virtual bool getInfo(const char* pInfoName, std::vector<double>& res);
  
+private:
+	struct NetCDFDim {
+		int    id;
+		char   name[NC_MAX_NAME+1];
+		size_t length;
+		bool   isUnlimited;
+	};
+
+	struct NetCDFAtt {
+		int     id;
+		char    name[NC_MAX_NAME+1];
+		size_t  length;
+		nc_type type;
+	};
+
+	struct NetCDFVar {
+		int       id;
+		char      name[NC_MAX_NAME+1];
+		std::vector<int> dimids;
+		std::vector<NetCDFAtt> atts;
+		nc_type   type;
+		void* buffer;
+		int valuesize;
+	};
+
+	struct NetCDFFile {
+		int       id;
+		std::vector<NetCDFDim> dims;
+		std::vector<NetCDFVar> vars;
+		std::vector<NetCDFAtt> atts;
+	};
+
+	NetCDFFile _file;
+
+	void resetFile();
+
+	bool loadNetCDFFileInfo();
+
+	int getNbVarRecords(const NetCDFVar& var);
+
+	int getNbVarValuesByRecord(const NetCDFVar& var);
+
+	bool loadData(NetCDFVar& var);
+
+	std::string getErrorMessage(int status);
 };
  
 } /* LocalFileInterface */
@@ -115,7 +115,7 @@ public:
 			lVI->setFilesFormat(VIFileFormat::FORMAT_CDF);
 		else if (loadedVIFormat == "VOT")
 			lVI->setFilesFormat(VIFileFormat::FORMAT_VOT);
-		else if (loadedVIFormat == "NETCDF")
+		else if (loadedVIFormat == "NC")
 			lVI->setFilesFormat(VIFileFormat::FORMAT_NETCDF);
 		else
 		{
@@ -41,6 +41,13 @@ log4cxx::LoggerPtr ParameterManager::_logger(
  */
 boost::mutex ParameterManager::mutexCDFLib;
  
+/*
+ * @brief Mutex used to protect multi-thread access to NetCDF lib (NetCDF lib is not thread-safe!)
+ * This mutex is a part of the ParameterManager to be shared between by FileWriterNetCDF (DownloadOutput plugin)
+ * and class and FileReaderNetCDF (ParamGetLocalFile)
+ */
+boost::mutex ParameterManager::mutexNetCDFLib;
+
 #define DEFAULT_GAP_THRESHOLD_VALUE 5
  
 ParameterManager::ParameterManager() :
@@ -55,6 +55,13 @@ namespace AMDA
        */
       static boost::mutex mutexCDFLib;
  
+      /*
+       * @brief Mutex used to protect multi-thread access to netCDF lib (netCDF lib is not thread-safe!)
+       * This mutex is a part of the ParameterManager to be shared between by FileWriterNetCDF (DownloadOutput plugin)
+       * and class and FileReaderNetCDF (ParamGetLocalFile)
+       */
+      static boost::mutex mutexNetCDFLib;
+
       /**
        * @return no return, do a getParameter
        */
 <?xml version="1.0" encoding="UTF-8"?>
 <param xml:id="local_netcdf_b">
   <info>
-	<name>Mag Field vector</name>
-	<short_name>Mag Field</short_name>
-	<components>Bx,By,Bz</components>
+	<name>B Mag</name>
+	<short_name>|B|</short_name>
+	<components></components>
 	<units>nT</units>
 	<coordinates_system></coordinates_system>
 	<tensor_order>1</tensor_order>
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<param xml:id="local_netcdf_v">
+  <info>
+	<name>Velocity</name>
+	<short_name>V</short_name>
+	<components></components>
+	<units>km/s</units>
+	<coordinates_system></coordinates_system>
+	<tensor_order>1</tensor_order>
+	<si_conversion></si_conversion>
+	<fill_value></fill_value>
+	<ucd></ucd>
+	<dataset_id></dataset_id>
+  </info>
+  <get>
+  <localvi id="15">
+  	<param id="V" minSampling="3600" maxSampling="3600"/>	
+  </localvi>
+  </get>
+  <process/>
+  <output/>
+</param>
@@ -87,7 +87,7 @@
 		<file name="mms1_scm_brst_l2_scb_20150815130334_v1.1.0.cdf" start="1439643815.000000"
 			stop="1439643925.000000" />
 	</vi>
-	<vi id="15" format="NETCDF" start="662691600.000000" stop="694223999.000000">
+	<vi id="15" format="NC" start="662691600.000000" stop="694223999.000000">
 		<file name="venus_1991.nc" start="662691600.000000"
 			stop="694223999.000000" />
 	</vi>