/** * * Created on: 03 oct. 2014 * Author: AKKA */ #include "FileWriterASCIIAbstract.hh" #include "TimeUtil.hh" #include "ParamData.hh" #include #include #include #include #include namespace AMDA { namespace ParamOutputImpl { namespace Download { namespace FileWriter { #define ASCII_TIME_ID "AMDA_TIME" FileWriterASCIIAbstract::FileWriterASCIIAbstract(AMDA::Parameters::ParameterManager& pParameterManager) : FileWriter(pParameterManager), _outputFilePath(""), _writeInfo(true) { //Environment variable used to hide all info blocs char *hideHeader; if((hideHeader = getenv("HIDE_HEADER_FILE")) != NULL) if (strcmp(hideHeader,"true") == 0) _writeInfo = false; } FileWriterASCIIAbstract::~FileWriterASCIIAbstract(void) { } bool FileWriterASCIIAbstract::createNewFile(std::string filePath) { //close previous opened file closeFile(); //open file stream _outputFile.open(filePath); if (!_outputFile.is_open()) return false; //write begin file tag writeBeginFile(); //write begin description tag if needed if (_writeInfo) writeBeginGeneralDescription(); _outputFilePath = filePath; return true; } void FileWriterASCIIAbstract::closeFile(void) { if (_outputFile.is_open()) { //write end file tag writeEndFile(); //close file stream _outputFile.flush(); _outputFile.close(); //re-init writer _outputFilePath = ""; cleanTempFiles(); _outputParamsList.clear(); } } bool FileWriterASCIIAbstract::addParameter(ParamProperties* paramProp, AMDA::Common::ParameterIndexComponentList &indexList, FileDataType type, bool isFirstParam, int dim1Size, int dim2Size) { if (!_outputFile.is_open() || !paramProp) return false; if (isFirstParam) { //add time field description _fieldInfoMap[ASCII_TIME_ID].name = "Time"; _fieldInfoMap[ASCII_TIME_ID].ucd = "time.epoch"; _fieldInfoMap[ASCII_TIME_ID].type = DT_TIME; _fieldInfoMap[ASCII_TIME_ID].dimSize1 = 1; _fieldInfoMap[ASCII_TIME_ID].dimSize2 = 1; _fieldInfoMap[ASCII_TIME_ID].componentsList.clear(); } //get current parameter AMDA::Parameters::ParameterSPtr crtParam = _parameterManager.getParameter(paramProp->getOutputId()); if (crtParam == nullptr) return false; //get current param info AMDA::Info::ParamInfoSPtr paramInfo = AMDA::Info::ParamMgr::getInstance()->getParamInfoFromId(crtParam->getInfoId()); //init field working structure _fieldInfoMap[paramProp->getOutputId()].name = paramInfo->getName(); _fieldInfoMap[paramProp->getOutputId()].ucd = paramInfo->getUcd(); _fieldInfoMap[paramProp->getOutputId()].type = type; _fieldInfoMap[paramProp->getOutputId()].dimSize1 = dim1Size; _fieldInfoMap[paramProp->getOutputId()].dimSize2 = dim2Size; _fieldInfoMap[paramProp->getOutputId()].componentsList.clear(); for (auto component : indexList) _fieldInfoMap[paramProp->getOutputId()].componentsList.push_back(component); //temporary file for this parameter std::string paramFileName = _outputFilePath; paramFileName += "_"; paramFileName += paramProp->getOutputId(); _fieldInfoMap[paramProp->getOutputId()].filePath = paramFileName; _fieldInfoMap[paramProp->getOutputId()].file = new std::fstream(); _fieldInfoMap[paramProp->getOutputId()].file->open(paramFileName, std::fstream::out); if (!_fieldInfoMap[paramProp->getOutputId()].file->is_open()) return false; //ordered list of parameters _outputParamsList.push_back(paramProp->getOutputId()); return true; } bool FileWriterASCIIAbstract::isInfoInSeparateFile(bool onlyOneInterval, OutputStructure outputStructure) { if (onlyOneInterval) return false; return ((outputStructure == ONE_FILE_PER_INTERVAL) || (outputStructure == ONE_FILE_PER_INTERVAL_REFPARAM)|| (outputStructure == ONE_FILE_PER_PARAMETER_PER_INTERVAL)); } bool FileWriterASCIIAbstract::writeError(std::string msg) { if (!_outputFile.is_open()) return false; writeErrorInfo(msg); if (_writeInfo) writeEndGeneralDescription(); return true; } bool FileWriterASCIIAbstract::writeAMDAInfo(std::string version, std::string createdby, std::string acknowledgement) { writeBeginInfoGroup("AMDA INFO",0); writeSingleInfo(AMDA_ABOUT, createdby, 0); writeSingleInfo(AMDA_VERSION, version, 0); writeSingleInfo(AMDA_ACKNOWLEDGEMENT, acknowledgement, 0); writeEndInfoGroup(0); return true; } bool FileWriterASCIIAbstract::writeRequestInfo(std::string structure, std::string timeFormat, int timeResolution, std::string outputParams, std::string ttName) { writeBeginInfoGroup("REQUEST INFO",0); writeSingleInfo(REQUEST_STRUCTURE, structure, 0); writeSingleInfo(REQUEST_TIMEFORMAT, timeFormat, 0); if (timeResolution > 0) { std::stringstream timeRes; timeRes << timeResolution; writeSingleInfo(REQUEST_TIMERES, timeRes.str(), 0); } if (!ttName.empty()) writeSingleInfo(REQUEST_TTNAME, ttName, 0); writeSingleInfo(REQUEST_OUTPUTPARAMS, outputParams, 0); writeEndInfoGroup(0); return true; } bool FileWriterASCIIAbstract::writeMissionInfo(MissionInfoSPtr missionInfo, std::set,FileWriter::cmpParamInfoStruct>& baseParamsInfoList) { std::string missionId; writeBeginInfoGroup("",0); if (missionInfo == nullptr) { writeSingleInfo(MISSION_ID, "NONE", 0); missionId = ""; } else { //write info about mission std::vector> infoMap = missionInfo->getInfoMap(); for (auto info : infoMap) writeSingleInfo(info.first, info.second, 0); missionId = missionInfo->getId(); } //build list of instruments to write bool writeInstrumentWithoutInfo = false; std::set, FileWriter::cmpInstrumentInfoStruct> instrumentInfoList; for (auto paramInfo : baseParamsInfoList) { DataSetInfoSPtr dataSetInfo = DataSetMgr::getInstance()->getDataSetInfoFromId(paramInfo.second->getDatasetId()); if (!dataSetInfo) continue; InstrumentInfoSPtr instrumentInfo = InstrumentMgr::getInstance()->getInstrumentInfoFromId(dataSetInfo->getInstrumentId()); if (!instrumentInfo) { if (missionId.empty()) writeInstrumentWithoutInfo = true; continue; } MissionInfoSPtr mission = MissionMgr::getInstance()->getMissionInfoFromId(instrumentInfo->getMissionId()); if ((!mission && missionId.empty()) || (mission && (mission->getId() == missionId))) { bool toAdd = true; for (auto existingInstrumentInfo : instrumentInfoList) { if (existingInstrumentInfo.first.compare(instrumentInfo->getId()) == 0) { toAdd = false; break; } } if (toAdd) instrumentInfoList.insert(std::pair(instrumentInfo->getId(), instrumentInfo)); } } //write instruments info writeBeginInfoList("INSTRUMENTS", 0); for (auto instrumentInfo : instrumentInfoList) writeInstrumentInfo(instrumentInfo.second, baseParamsInfoList); if (writeInstrumentWithoutInfo) writeInstrumentInfo(InstrumentInfoSPtr(), baseParamsInfoList); writeEndInfoList(); writeEndInfoGroup(0); return true; } bool FileWriterASCIIAbstract::writeInstrumentInfo(InstrumentInfoSPtr instrumentInfo, std::set,FileWriter::cmpParamInfoStruct>& baseParamsInfoList) { std::string instrumentId; writeBeginInfoGroup("",1); if (instrumentInfo == nullptr) { writeSingleInfo(INSTRUMENT_ID, "NONE", 1); instrumentId = ""; } else { //write info about instrument std::vector> infoMap = instrumentInfo->getInfoMap(); for (auto info : infoMap) writeSingleInfo(info.first, info.second, 1); instrumentId = instrumentInfo->getId(); } //write list of dataset info to write bool writeDatasetWithoutInfo = false; std::set, FileWriter::cmpDatasetInfoStruct> datasetInfoList; for (auto paramInfo : baseParamsInfoList) { DataSetInfoSPtr dataSetInfo = DataSetMgr::getInstance()->getDataSetInfoFromId(paramInfo.second->getDatasetId()); if (!dataSetInfo) { if (instrumentId.empty()) writeDatasetWithoutInfo = true; continue; } InstrumentInfoSPtr instrumentInfo = InstrumentMgr::getInstance()->getInstrumentInfoFromId(dataSetInfo->getInstrumentId()); if ((!instrumentInfo && instrumentId.empty()) || (instrumentInfo && (instrumentInfo->getId() == instrumentId))) { bool toAdd = true; for (auto existingDatasetInfo : datasetInfoList) { if (existingDatasetInfo.first.compare(dataSetInfo->getId()) == 0) { toAdd = false; break; } } if (toAdd) datasetInfoList.insert(std::pair(dataSetInfo->getId(), dataSetInfo)); } } //write datasets info writeBeginInfoList("DATASETS", 1); for (auto datasetInfo : datasetInfoList) writeDatasetInfo(datasetInfo.second, baseParamsInfoList); if (writeDatasetWithoutInfo) writeDatasetInfo(DataSetInfoSPtr(), baseParamsInfoList); writeEndInfoList(); writeEndInfoGroup(1); return true; } bool FileWriterASCIIAbstract::writeDatasetInfo(DataSetInfoSPtr datasetInfo, std::set,FileWriter::cmpParamInfoStruct>& baseParamsInfoList) { std::string datasetId; writeBeginInfoGroup("", 2); if (datasetInfo == nullptr) { writeSingleInfo(DATASET_ID, "NONE", 2); datasetId = ""; } else { //write info about datasetId std::vector> infoMap = datasetInfo->getInfoMap(); for (auto info : infoMap) writeSingleInfo(info.first, info.second, 2); datasetId = datasetInfo->getId(); } //build parameters info to write std::set,FileWriter::cmpParamInfoStruct> paramInfoList; for (auto paramInfo : baseParamsInfoList) { DataSetInfoSPtr dataSetInfo = DataSetMgr::getInstance()->getDataSetInfoFromId(paramInfo.second->getDatasetId()); if ((!dataSetInfo && datasetId.empty()) || (dataSetInfo && (dataSetInfo->getId() == datasetId))) paramInfoList.insert(paramInfo); } //write parameters info writeBeginInfoList("PARAMETERS", 2); for (auto paramInfo : paramInfoList) { writeParameterInfo(paramInfo.second, 3); } writeEndInfoList(); writeEndInfoGroup(2); return true; } bool FileWriterASCIIAbstract::writeParameterInfo(ParamInfoSPtr parameterInfo, int level) { if (parameterInfo == nullptr) return false; std::vector> infoMap = parameterInfo->getInfoMap(&_parameterManager); writeBeginInfoGroup("",level); for (auto info : infoMap) writeSingleInfo(info.first, info.second, level); writeEndInfoGroup(level); return true; } bool FileWriterASCIIAbstract::writeParamsInfo(ParamPropertiesList& paramPropertiesList, OutputStructure outputStructure, std::string currentParamId) { std::set,FileWriter::cmpParamInfoStruct> baseParamsInfoList; std::set,FileWriter::cmpParamInfoStruct> derivedParamsInfoList; //sort parameters info for (auto paramProperties : paramPropertiesList) { if ((outputStructure == ONE_FILE_PER_PARAMETER_PER_INTERVAL) && (currentParamId != paramProperties->getOutputId())) //add info only for current parameter continue; sortParametersInfoByType(paramProperties->getOutputId(), baseParamsInfoList, derivedParamsInfoList); } //write derived parameters info if (!derivedParamsInfoList.empty()) { writeBeginInfoGroup("DERIVED PARAMETERS",0); writeBeginInfoList("PARAMETERS",0); for (auto paramInfo : derivedParamsInfoList) writeParameterInfo(paramInfo.second,0); writeEndInfoList(); writeEndInfoGroup(0); } //get list of all missions std::set, FileWriter::cmpMissionInfoStruct> missionInfoList; bool atLeastOneParamWithoutMissionInfo = false; for (auto paramInfo : baseParamsInfoList) { DataSetInfoSPtr dataSetInfo = DataSetMgr::getInstance()->getDataSetInfoFromId(paramInfo.second->getDatasetId()); if (!dataSetInfo) { atLeastOneParamWithoutMissionInfo = true; continue; } InstrumentInfoSPtr instrumentInfo = InstrumentMgr::getInstance()->getInstrumentInfoFromId(dataSetInfo->getInstrumentId()); if (!instrumentInfo) { atLeastOneParamWithoutMissionInfo = true; continue; } MissionInfoSPtr mission = MissionMgr::getInstance()->getMissionInfoFromId(instrumentInfo->getMissionId()); if (!mission) { atLeastOneParamWithoutMissionInfo = true; continue; } bool toAdd = true; for (auto existingMissionInfo : missionInfoList) { if (existingMissionInfo.first.compare(mission->getId()) == 0) { toAdd = false; break; } } if (toAdd) missionInfoList.insert(std::pair(mission->getId(), mission)); } //write the base parameters tree writeBeginInfoGroup("BASE PARAMETERS",0); //write missions writeBeginInfoList("MISSIONS", 0); for (auto missionInfo : missionInfoList) { //Add mission info writeMissionInfo(missionInfo.second, baseParamsInfoList); } //if at least one parameter without mission, write NO MISSION INFO if (atLeastOneParamWithoutMissionInfo) writeMissionInfo(MissionInfoSPtr(), baseParamsInfoList); writeEndInfoList(); writeEndInfoGroup(0); return true; } bool FileWriterASCIIAbstract::writeIntervalInfo(std::string startTime, std::string stopTime) { writeBeginInfoGroup("INTERVAL INFO",0); writeSingleInfo(INTERVAL_START, startTime, 0); writeSingleInfo(INTERVAL_STOP, stopTime, 0); writeEndInfoGroup(0); return true; } bool FileWriterASCIIAbstract::writeTimeData(std::string paramId, double data, OutputFormatTime timeFormat, bool isFirstParam) { if (!isFirstParam) return true; //time already write for this record, no error std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; (*crtFile) << getDataStartTag(true); switch (timeFormat) { case FORMAT_OUTPUT_TIME_ISO: TimeUtil::formatTimeDateInIso(data, *crtFile); break; case FORMAT_OUTPUT_TIME_DD: //two fill characters to preserve tests validation (*crtFile) << getDataFillCharacter() << getDataFillCharacter(); TimeUtil::double2DD_TimeDate(data, *crtFile); break; case FORMAT_OUTPUT_TIME_DOUBLE: default: //one fill character to preserve tests validation (*crtFile) << getDataFillCharacter() << std::scientific << std::setprecision(8) << data; break; } (*crtFile) << getDataStopTag(); return true; } bool FileWriterASCIIAbstract::writeFloatData(std::string paramId, int varIndex, float data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); crtFile->precision(5); (*crtFile) << getDataFillCharacter(); crtFile->fill(' '); if (!precision) { crtFile->width(7); crtFile->setf(std::ios::fixed, std::ios::floatfield); crtFile->setf(std::ios::right, std::ios::adjustfield); crtFile->precision(3); } if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; return true; } bool FileWriterASCIIAbstract::writeShortData(std::string paramId, int varIndex, short data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); (*crtFile) << getDataFillCharacter(); crtFile->fill(' '); if (!precision) crtFile->width(7); if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; return true; } bool FileWriterASCIIAbstract::writeIntData(std::string paramId, int varIndex, int data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); (*crtFile) << getDataFillCharacter(); crtFile->fill(' '); if (!precision) crtFile->width(7); if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; return true; } bool FileWriterASCIIAbstract::writeDoubleData(std::string paramId, int varIndex, double data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); //five fill characters to preserve tests validation (*crtFile) << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter(); crtFile->fill(' '); if (!precision) { crtFile->width(10); crtFile->setf(std::ios::right, std::ios::adjustfield); crtFile->precision(3); //crtFile->precision(std::numeric_limits::digits10 + 1); if (isNAN(data)) (*crtFile) << std::fixed << getNanString(); else (*crtFile) << std::fixed << data; } else { if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; } return true; } bool FileWriterASCIIAbstract::writeLongDoubleData(std::string paramId, int varIndex, long double data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); //five fill characters to preserve tests validation (*crtFile) << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter() << getDataFillCharacter(); crtFile->fill(' '); if (!precision) { crtFile->width(10); crtFile->setf(std::ios::right, std::ios::adjustfield); crtFile->precision(3); } if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; return true; } bool FileWriterASCIIAbstract::writeLogicalData(std::string paramId, int varIndex, AMDA::Parameters::LogicalData data, bool precision) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; if (varIndex == 0) (*crtFile) << getDataStartTag(false); else (*crtFile) << getDataValueSeparator(); (*crtFile) << getDataFillCharacter(); (*crtFile).fill(' '); if (!precision) crtFile->width(6); if (isNAN(data)) (*crtFile) << getNanString(); else (*crtFile) << data; return true; } bool FileWriterASCIIAbstract::goToNextRecord(std::string paramId) { std::fstream *crtFile = _fieldInfoMap[paramId].file; if (crtFile == NULL) return false; (*crtFile) << getDataStopTag() << std::endl; return true; } bool FileWriterASCIIAbstract::finalize(bool isInfoFile) { if (!_outputFile.is_open()) { cleanTempFiles(); return false; } if (isInfoFile && _writeInfo) { writeEndGeneralDescription(); return true; } if (_fieldInfoMap.empty()) { cleanTempFiles(); return false; } //set position to the beginning of all parameters files for (auto paramId : _outputParamsList) { if (!_fieldInfoMap[paramId].file->is_open()) { cleanTempFiles(); return false; } _fieldInfoMap[paramId].file->flush(); _fieldInfoMap[paramId].file->close(); //reopen the file in read mode _fieldInfoMap[paramId].file->open(_fieldInfoMap[paramId].filePath, std::fstream::in); } if (_writeInfo) { writeEndGeneralDescription(); writeBeginFieldsDescription(); writeFieldDescription(ASCII_TIME_ID); for (auto paramId : _outputParamsList) writeFieldDescription(paramId); writeEndFieldsDescription(); } //Merge result and push it in output file writeBeginData(); std::string loadedLine; bool finished = false; do { std::stringstream resultLine; for (auto paramId : _outputParamsList) { std::getline((*_fieldInfoMap[paramId].file), loadedLine); if (finished && !(*_fieldInfoMap[paramId].file).eof()) { //files seems to not have the same number of lines!! cleanTempFiles(); return false; } finished = (*_fieldInfoMap[paramId].file).eof(); resultLine << loadedLine; } if (!resultLine.str().empty()) writeDataRecord(resultLine.str()); } while (!finished); cleanTempFiles(); writeEndData(); return true; } void FileWriterASCIIAbstract::cleanTempFiles() { for (auto fieldInfo : _fieldInfoMap) { if (fieldInfo.second.file != NULL) { if (fieldInfo.second.file->is_open()) { fieldInfo.second.file->flush(); fieldInfo.second.file->close(); } delete fieldInfo.second.file; fieldInfo.second.file = NULL; } } for (auto fieldInfo : _fieldInfoMap) remove(fieldInfo.second.filePath.c_str()); _fieldInfoMap.clear(); _outputParamsList.clear(); } } /* namespace FileWriter */ } /* namespace Download */ } /* namespace ParamOutputImpl */ } /* namespace AMDA */