diff --git a/src/ExternLib/DataFiltering/DataFiltering.hh b/src/ExternLib/DataFiltering/DataFiltering.hh index 3e77ffd..0b9fe8b 100644 --- a/src/ExternLib/DataFiltering/DataFiltering.hh +++ b/src/ExternLib/DataFiltering/DataFiltering.hh @@ -134,16 +134,18 @@ namespace AMDA { } void write(ParamDataIndexInfo &pParamDataIndexInfo) { - std::fstream fs; - fs.open("dataFiltering0D.log", std::fstream::in | std::fstream::out | std::fstream::app); - if(pParamDataIndexInfo._startIndex ==0 && pParamDataIndexInfo._nbDataToProcess >0){ - std::chrono::system_clock::time_point today = std::chrono::system_clock::now(); - std::time_t tt; - tt = std::chrono::system_clock::to_time_t ( today ); - fs <<"# "<< std::ctime(&tt)<<"# [Debug] outPut of dataFiltering plugin"< 0) { + std::chrono::system_clock::time_point today = std::chrono::system_clock::now(); + std::time_t tt; + tt = std::chrono::system_clock::to_time_t(today); + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering0D plugin" << std::endl; + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl; + fs << "# time" << " " << "removed_value" << " average" << " " << "std" << " " << "factor" << std::endl; + } + bool isFirstSet = true; + for (unsigned int _index = pParamDataIndexInfo._startIndex; _index < pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess; ++_index) { @@ -165,17 +167,21 @@ namespace AMDA { // traitement une fois le container rempli _container._ave = (ElemType) _container._sum / _container._size; _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size); - + int offSet = _nPoints / 4; + if (isFirstSet) + offSet = 0; // filter Elements - std::list::iterator itTime = _container._times.begin(); - for (auto it = _container._values.begin(); it != _container._values.end(); ++it) { + std::list::iterator itTime = std::next(_container._times.begin(), offSet); + auto startVal = std::next(_container._values.begin(), offSet); + auto endVal = std::next(_container._values.begin(), 2 * offSet); + for (auto it = startVal; it != endVal; ++it) { ElemType crt_val = *it; if (isNAN(crt_val)) continue; if (std::abs(crt_val - _container._ave) > _factor * _container._sig) { - fs<::iterator itTime = _container._times.begin(); + std::list::iterator itTime = _container._times.begin(); for (auto it = _container._values.begin(); it != _container._values.end(); ++it) { ElemType crt_val = *it; if (std::abs(crt_val - _container._ave) > _factor * _container._sig) { - fs<pushTime(*itTime); _paramOutput->getDataList().push_back(crt_val); - ++ itTime; + ++itTime; } } } - // clear container - _container._ave = 0; - _container._sig = 0; - _container._size = 0; - _container._sum =0; - _container._times.clear(); - _container._sumsq = 0; - _container._values.clear(); - fs.close(); + // clear container + _container._ave = 0; + _container._sig = 0; + _container._size = 0; + _container._sum = 0; + _container._times.clear(); + _container._sumsq = 0; + _container._values.clear(); + fs.close(); } @@ -282,6 +289,17 @@ namespace AMDA { } void write(ParamDataIndexInfo &pParamDataIndexInfo) { + std::fstream fs; + fs.open("dataFiltering1D.log", std::fstream::in | std::fstream::out | std::fstream::app); + if (pParamDataIndexInfo._startIndex == 0 && pParamDataIndexInfo._nbDataToProcess > 0) { + std::chrono::system_clock::time_point today = std::chrono::system_clock::now(); + std::time_t tt; + tt = std::chrono::system_clock::to_time_t(today); + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering1D plugin" << std::endl; + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl; + fs << "#index" << " " << "time" << " " << "removed_value" << " average" << " " << "std" << " " << "factor" << std::endl; + } + bool isFirstSet = true; int size = 0; for (unsigned int _index = pParamDataIndexInfo._startIndex; _index < pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess; @@ -290,7 +308,7 @@ namespace AMDA { std::vector inputElt = _paramInput.get(_index); if (_container->_values.empty()) { - size = inputElt.size(); + size = inputElt.size(); _container->init(size); } // initialiser et remplir container @@ -312,7 +330,14 @@ namespace AMDA { } // filter Elements - for (auto elt = _container->_values.begin(); elt != _container->_values.end(); ++elt) { + int offSet = _nPoints / 4; + if (isFirstSet) + offSet = 0; + // filter Elements + std::list::iterator itTime = std::next(_container->_times.begin(), offSet); + auto startVal = std::next(_container->_values.begin(), offSet); + auto endVal = std::next(_container->_values.begin(), 2 * offSet); + for (auto elt = startVal; elt != endVal; ++elt) { int i; for (auto it = elt->begin(); it != elt->end(); ++it) { i = it - elt->begin(); @@ -322,6 +347,9 @@ namespace AMDA { } if (std::abs(crt_val - _container->_ave[i]) > _factor * _container->_sig[i]) { + fs << i << " " << TimeUtil::formatTimeDateInIso(*itTime); + fs <<" "<< std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i]; + fs << " " << _factor << std::endl; _container->_sum[i] -= crt_val; _container->_sumsq[i] -= crt_val*crt_val; _container->_size[i] -= 1; @@ -329,6 +357,7 @@ namespace AMDA { *it << NotANumber(); } } + ++itTime; } _paramOutput->pushTime(_container->_times.front()); _paramOutput->getDataList().push_back(_container->_values.front()); @@ -348,6 +377,7 @@ namespace AMDA { } _container->_values.push_back(inputElt); _container->_times.push_back(crtTime); + isFirstSet = false; } // last value filter and write if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1 && !_container->_values.empty()) { @@ -355,6 +385,8 @@ namespace AMDA { _container->_ave[i] = (ElemType) _container->_sum[i] / _container->_size[i]; _container->_sig[i] = std::sqrt((_container->_sumsq[i] - _container->_sum[i] * _container->_sum[i] / _container->_size[i]) / _container->_size[i]); } + + std::list::iterator itTime = _container->_times.begin(); for (auto elt = _container->_values.begin(); elt != _container->_values.end(); ++elt) { int i; for (auto it = elt->begin(); it != elt->end(); ++it) { @@ -365,6 +397,9 @@ namespace AMDA { } if (std::abs(crt_val - _container->_ave[i]) > _factor * _container->_sig[i]) { + fs << i << " " << TimeUtil::formatTimeDateInIso(*itTime); + fs << std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i]; + fs << " " << _factor << std::endl; _container->_sum[i] -= crt_val; _container->_sumsq[i] -= crt_val*crt_val; _container->_size[i] -= 1; @@ -372,17 +407,13 @@ namespace AMDA { *it << NotANumber(); } } - } - - std::list::iterator itTimes = _container->_times.begin(); - for (auto elt : _container->_values) { - _paramOutput->pushTime(*itTimes); - _paramOutput->getDataList().push_back(elt); - ++itTimes; + _paramOutput->pushTime(*itTime); + _paramOutput->getDataList().push_back(*elt); + ++itTime; } } } - + fs.close(); } private: /** @@ -429,6 +460,17 @@ namespace AMDA { } void write(ParamDataIndexInfo &pParamDataIndexInfo) { + std::fstream fs; + fs.open("dataFiltering1D.log", std::fstream::in | std::fstream::out | std::fstream::app); + if (pParamDataIndexInfo._startIndex == 0 && pParamDataIndexInfo._nbDataToProcess > 0) { + std::chrono::system_clock::time_point today = std::chrono::system_clock::now(); + std::time_t tt; + tt = std::chrono::system_clock::to_time_t(today); + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering2D plugin" << std::endl; + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl; + fs << "#index1 Index2 time removed_value average std factor" << std::endl; + } + bool isFirstSet = true; // init container int size1 = 0; int size2 = 0; @@ -465,9 +507,15 @@ namespace AMDA { _container->_ave[i] = (ElemType) _container->_sum[i] / _container->_size[i]; _container->_sig[i] = std::sqrt((_container->_sumsq[i] - _container->_sum[i] * _container->_sum[i] / _container->_size[i]) / _container->_size[i]); } - + // filter elements + int offSet = _nPoints / 4; + if (isFirstSet) + offSet = 0; // filter Elements - for (auto elt = _container->_2Dvalues.begin(); elt != _container->_2Dvalues.end(); ++elt) { + std::list::iterator itTime = std::next(_container->_times.begin(), offSet); + auto startVal = std::next(_container->_2Dvalues.begin(), offSet); + auto endVal = std::next(_container->_2Dvalues.begin(), 2 * offSet); + for (auto elt = startVal; elt != endVal; ++elt) { Tab2DData outputElt(size1, size2); Tab2DData inputElt = *elt; for (int i = 0; i < size1; i++) { @@ -478,6 +526,9 @@ namespace AMDA { if (isNAN(crt_val)) { outputElt[i][j] = res; } else if (std::abs(crt_val - _container->_ave[i * size2 + j]) > _factor * _container->_sig[i * size2 + j]) { + fs << i << " " << j << " " << TimeUtil::formatTimeDateInIso(*itTime); + fs <<" "<< std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i]; + fs << " " << _factor << std::endl; _container->_sum[i * size2 + j] -= crt_val; _container->_sumsq[i * size2 + j] -= crt_val*crt_val; _container->_size[i * size2 + j] -= 1; @@ -513,6 +564,7 @@ namespace AMDA { } _container->_2Dvalues.push_back(inputElt); _container->_times.push_back(crtTime); + isFirstSet = false; } // last value filter and write @@ -525,7 +577,7 @@ namespace AMDA { // filter values and write // filter Elements - std::list::iterator itTimes = _container->_times.begin(); + std::list::iterator itTime = _container->_times.begin(); for (auto elt = _container->_2Dvalues.begin(); elt != _container->_2Dvalues.end(); ++elt) { Tab2DData outputElt(size1, size2); Tab2DData inputElt = *elt; @@ -533,6 +585,9 @@ namespace AMDA { for (int j = 0; j < size2; j++) { ElemType crt_val = inputElt[i][j]; if (std::abs(crt_val - _container->_ave[i * size2 + j]) > _factor * _container->_sig[i * size2 + j]) { + fs << i << " " << j << " " << TimeUtil::formatTimeDateInIso(*itTime); + fs << std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i]; + fs << " " << _factor << std::endl; _container->_sum[i * size2 + j] -= crt_val; _container->_sumsq[i * size2 + j] -= crt_val*crt_val; _container->_size[i * size2 + j] -= 1; @@ -546,15 +601,15 @@ namespace AMDA { } // write *elt = outputElt; - _paramOutput->pushTime(*itTimes); + _paramOutput->pushTime(*itTime); _paramOutput->getDataList().push_back(outputElt); outputElt.free(); - ++itTimes; + ++itTime; } } } - + fs.close(); } private: /* -- libgit2 0.21.2