/* * FileReaderNetCDF.cc * * Created on: Feb 08, 2018 * Author: AKKA */ #include "FileReaderNetCDF.hh" #include "ParameterManager.hh" #include "LocalFileInterfaceConfig.hh" #include "TimeUtil.hh" #include namespace AMDA { namespace LocalFileInterface { FileReaderNetCDF::FileReaderNetCDF() { resetFile(); } FileReaderNetCDF::~FileReaderNetCDF() { resetFile(); } bool FileReaderNetCDF::open(std::string filePath) { boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib); if (isOpened()) { return true; } int status = nc_open(filePath.c_str(), 0, &_file.id); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::open - Cannot open netCDF file : " << getErrorMessage(status)) resetFile(); return false; } return loadNetCDFFileInfo(); } bool FileReaderNetCDF::close(void) { boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib); if (!isOpened()) { return true; } int status = nc_close(_file.id); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::close - Cannot close netCDF file : " << getErrorMessage(status)) return false; } resetFile(); return true; } bool FileReaderNetCDF::isOpened(void) { return (_file.id != 0); } std::string FileReaderNetCDF::getTimeParamId(void) { if (!isOpened()) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getTimeParamId - File not opened") return ""; } for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { std::string name = it->name; std::transform(name.begin(), name.end(), name.begin(), ::tolower); if ((std::strcmp(name.c_str(),"time") == 0) && (it->type == NC_DOUBLE)) { return it->name; } } LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getTimeParamId - Cannot find the time variable") return ""; } bool FileReaderNetCDF::getParamInfo(std::string& paramId, LocalParamType& paramType, int& dim1Size, int& dim2Size) { for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { if (std::strcmp(it->name, paramId.c_str()) == 0) { if (it->dimids.size() == 1) { //scalar dim1Size = 1; dim2Size = 1; } else if (it->dimids.size() == 2) { //vector dim1Size = _file.dims[it->dimids[1]].length; dim2Size = 1; } else if (it->dimids.size() == 3) { //Tab2D dim1Size = _file.dims[it->dimids[1]].length; dim2Size = _file.dims[it->dimids[2]].length; } else { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamInfo - Variable dimensions not supported") return false; } switch (it->type) { case NC_FLOAT : paramType = LocalParamType::TYPE_FLOAT; break; case NC_DOUBLE : paramType = LocalParamType::TYPE_DOUBLE; break; case NC_SHORT : paramType = LocalParamType::TYPE_SHORT; break; case NC_INT : paramType = LocalParamType::TYPE_INT; break; default: paramType = LocalParamType::TYPE_UNKNOWN; LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamInfo - Unknown variable type") return false; } } } return true; } int FileReaderNetCDF::getRecordIndex(std::string& timeId, double time) { boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib); for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { if (std::strcmp(it->name, timeId.c_str()) == 0) { if (!loadData(*it)) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getRecordIndex - Cannot load time data") return -1; } double* buffer = (double*)it->buffer; for (int i = 0; i < getNbVarRecords(*it); ++i) { if (time <= buffer[i]) { return i; } } } } return -1; } FileReaderStatus FileReaderNetCDF::getParamPacketData(std::string& timeId, std::string& paramId, int recordIndex, double stopTime, LocalParamDataPacket *packet) { boost::mutex::scoped_lock scoped_lock(AMDA::Parameters::ParameterManager::mutexNetCDFLib); NetCDFVar* timeVar = NULL; NetCDFVar* paramVar = NULL; for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { if (std::strcmp(it->name, timeId.c_str()) == 0) { timeVar = &(*it); } if (std::strcmp(it->name, paramId.c_str()) == 0) { paramVar = &(*it); } } if ((timeVar == NULL) || (paramVar == NULL)) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Cannot retrieve time or param variable") return FRS_ERROR; } if (!loadData(*timeVar) || !loadData(*paramVar)) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Cannot load time or param data") return FRS_ERROR; } double* time_buffer = (double*)timeVar->buffer; bool packetFull = false; int nbValues = getNbVarValuesByRecord(*paramVar); for (int i = recordIndex; i < getNbVarRecords(*paramVar); ++i) { double crtTime = time_buffer[i]; void* pos = (void*)((char*)paramVar->buffer + paramVar->valuesize * nbValues * i); if (crtTime > stopTime) { return FRS_FINISH; } if (!packet->addData(crtTime, pos, packetFull)) { if (packetFull) return FRS_MORE; LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getParamPacketData - Error to add data in packet"); return FRS_ERROR; } } return FRS_EOF; } bool FileReaderNetCDF::getInfo(const char* pInfoName, std::vector& res) { for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { if (std::strcmp(it->name, pInfoName) == 0) { if (it->dimids.empty()) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getInfo - dims error") return false; } int nbValues = 1; for (int i = 0; i < (int)it->dimids.size(); ++i) { nbValues *= _file.dims[it->dimids[i]].length; } int valuesize = 0; switch (it->type) { case NC_FLOAT : valuesize = sizeof(float); break; case NC_DOUBLE : valuesize = sizeof(double); break; case NC_SHORT : valuesize = sizeof(short); break; case NC_INT : valuesize = sizeof(int); break; default: LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getInfo - var type not supported") return false; } int varid; int status = nc_inq_varid (_file.id, it->name, &varid); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getInfo - Cannot load variable data : " << getErrorMessage(status)) return false; } void* buffer = std::malloc(nbValues*valuesize); status = nc_get_var(_file.id, varid, buffer); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getInfo - Cannot load info data : " << getErrorMessage(status)) std::free(buffer); return false; } for (int i = 0; i < nbValues; ++i) { switch (it->type) { case NC_FLOAT : res.push_back(((float*)buffer)[i]); break; case NC_DOUBLE : res.push_back(((double*)buffer)[i]); break; case NC_SHORT : res.push_back(((short*)buffer)[i]); break; case NC_INT : res.push_back(((int*)buffer)[i]); break; default: return false; } } std::free(buffer); return true; } } LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::getInfo - Cannot retrieve data for " << pInfoName) return false; } void FileReaderNetCDF::resetFile() { _file.id = 0; _file.dims.clear(); for (std::vector::iterator it = _file.vars.begin(); it != _file.vars.end(); ++it) { if (it->buffer != NULL) { std::free(it->buffer); it->buffer = NULL; } } _file.vars.clear(); _file.atts.clear(); } bool FileReaderNetCDF::loadNetCDFFileInfo() { if (!isOpened()) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - File not opened") return false; } int status, ndims, nvars, ngatts, unlimdimid; status = nc_inq(_file.id, &ndims, &nvars, &ngatts, &unlimdimid); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load info in netCDF file : " << getErrorMessage(status)) return false; } //Load dims for (int i = 0; i < ndims; ++i) { NetCDFDim dim; status = nc_inq_dim(_file.id, i, dim.name, &dim.length); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load dimension info : " << getErrorMessage(status)) return false; } dim.isUnlimited = (i == unlimdimid); _file.dims.push_back(dim); } //Load global attributes for (int i = 0; i < ngatts; ++i) { NetCDFAtt att; status = nc_inq_attname(_file.id, NC_GLOBAL, i, att.name); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load global attribute name : " << getErrorMessage(status)) return false; } status = nc_inq_att(_file.id, NC_GLOBAL, att.name, &att.type, &att.length); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load global attribute info : " << getErrorMessage(status)) return false; } _file.atts.push_back(att); } //Load variables for (int i = 0; i < nvars; ++i) { int natts; int ndims; int dimids[NC_MAX_VAR_DIMS]; NetCDFVar var; var.buffer = NULL; status = nc_inq_var(_file.id, i, var.name, &var.type, &ndims, dimids, &natts); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable info : " << getErrorMessage(status)) return false; } if (ndims > 3) { //Not supported by the kernel continue; } for (int j = 0; j < ndims; ++j) { var.dimids.push_back(dimids[j]); } //Load variable attributes for (int j = 0; j < natts; ++j) { NetCDFAtt att; status = nc_inq_attname(_file.id, i, j, att.name); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable attribute : " << getErrorMessage(status)) return false; } status = nc_inq_att(_file.id, i, att.name, &att.type, &att.length); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadNetCDFFileInfo - Cannot load variable attribute : " << getErrorMessage(status)) return false; } var.atts.push_back(att); } _file.vars.push_back(var); } return true; } int FileReaderNetCDF::getNbVarValuesByRecord(const NetCDFVar& var) { if (var.dimids.size() == 1) { //scalar return 1; } else if (var.dimids.size() == 2) { //vector return _file.dims[var.dimids[1]].length; } else if (var.dimids.size() == 3) { //tab2d return _file.dims[var.dimids[1]].length * _file.dims[var.dimids[2]].length; } return 1; } int FileReaderNetCDF::getNbVarRecords(const NetCDFVar& var) { if (var.dimids.size() < 1) return 0; return _file.dims[var.dimids[0]].length; } bool FileReaderNetCDF::loadData(NetCDFVar& var) { if (var.buffer != NULL) { return true; } int nbRecords = getNbVarRecords(var); int nbValues = getNbVarValuesByRecord(var); int varid; int status = nc_inq_varid (_file.id, var.name, &varid); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadData - Cannot load variable data : " << getErrorMessage(status)) return false; } switch (var.type) { case NC_FLOAT : var.valuesize = sizeof(float); break; case NC_DOUBLE : var.valuesize = sizeof(double); break; case NC_SHORT : var.valuesize = sizeof(short); break; case NC_INT : var.valuesize = sizeof(int); break; default: return false; } var.buffer = std::malloc(nbRecords*nbValues*var.valuesize); status = nc_get_var(_file.id, varid, var.buffer); if (status != NC_NOERR) { LOG4CXX_ERROR(gLogger, "FileReaderNetCDF::loadData - Cannot load variable data : " << getErrorMessage(status)) std::free(var.buffer); var.buffer = NULL; return false; } return true; } std::string FileReaderNetCDF::getErrorMessage(int status) { if (status == NC_NOERR) { return "NO ERROR"; } return nc_strerror(status); } } /* LocalFileInterface */ } /* AMDA */