CDPP / AMDA_Kernel

Commit 9aea9f9da288db3ba56314977b9e8a5e0f26e498

Authored by Hacene SI HADJ MOHAND
1 parent 456dd0a2
Exists in master and in 69 other branches 10038, 10098, 5390, 5770, 7616, 7875, 8459, 8528, 9016, 9016Bis, 9017, 9017-new, 9049, 9084, 9259, 9326, 9347, 9383, 9719, 9719.new, AllLegends, CDF_ISTP, CDF_ISTP_HOPEFULLY_ALL_WORKING, FER_10400, FER_10412, FER_10663, FER_10765, FER_11000, FER_11100, FER_11242, FER_11277, FER_11500, FER_11537, FER_11588, FER_11651, FER_19836, FER_5344, FER_6031, FER_6598, FER_6966, FER_7318, FER_7618, FER_7857, FER_8455, FER_9629, FER_CR41, FER_CR42, FER_CR43, FER_TFCat, Histo1D2D, JUNO_JEDI, MAZ_10936, MAZ_6018, MAZ_9016_New, MAZ_FIX_9016, SpeasyGet, amdadev, amdadev_test, cu2022, das2, density_maps, develop, evol_9042, feature/9259, pickle, rm_6912, rm_8932, test-merge, ubuntu

scalar data ok

Showing 4 changed files with 146 additions and 43 deletions   Show diff stats
Inline Side-by-side
src/ExternLib/DataFiltering/DataFiltering.hh
  Diff comments   View file @9aea9f9
... ... @@ -14,6 +14,9 @@
14 14 #ifndef DATAFILTERING_HH
15 15 #define DATAFILTERING_HH
16 16  
  17 +#include <list>
  18 +#include <c++/4.8.2/bits/stl_vector.h>
  19 +
17 20 #include "Parameter.hh"
18 21 #include "ParamData.hh"
19 22 #include "DataTypeMath.hh"
... ... @@ -23,11 +26,13 @@
23 26 namespace AMDA {
24 27 namespace Parameters {
25 28  
26   - namespace DataFiltering{
  29 + namespace DataFiltering {
27 30  
28 31 template <typename ElemType>
29 32 struct FilteringContainer {
  33 + // size of real points(not nan and not spike)
30 34 int _size = 0;
  35 + int _nFilteredPoints = 0;
31 36 ElemType _sum = 0;
32 37 ElemType _sumsq = 0;
33 38 ElemType _ave = 0;
... ... @@ -37,11 +42,24 @@ namespace AMDA {
37 42 };
38 43  
39 44 template <typename ElemType>
  45 + struct FilteringContainer1D {
  46 + // size of real points(not nan and not spike)
  47 + std::vector<int> _size;
  48 + std::vector<int> _nFilteredPoints;
  49 + std::vector<ElemType> _sum;
  50 + std::vector<ElemType> _sumsq;
  51 + std::vector<ElemType> _ave;
  52 + std::vector<ElemType> _sig;
  53 + std::vector<std::list<ElemType> > _values;
  54 + std::list<double> _times;
  55 + };
  56 +
  57 + template <typename ElemType>
40 58 class DataFiltering : public Operation {
41 59 public:
42 60 // constructor
43 61  
44   - DataFiltering(Process& pProcess, ParamDataSpec<ElemType >& paramInput, int factor = Default::_factor, int nPoints = Default::_nPoints) :
  62 + DataFiltering(Process& pProcess, ParamDataSpec<ElemType >& paramInput, double factor = Default::_factor, int nPoints = Default::_nPoints) :
45 63 Operation(pProcess),
46 64 _paramInput(paramInput),
47 65 _paramOutput(new ParamDataSpec<ElemType>()),
... ... @@ -61,45 +79,73 @@ namespace AMDA {
61 79 ++_index) {
62 80 double crtTime = _paramInput.getTime(_index);
63 81 ElemType inputElt = _paramInput.get(_index);
64   -
  82 +
65 83 // initialiser et remplir container
66   - while (_container._size < _nPoints) {
67   -
68   - _container._values.push_back(inputElt);
69   - _container._times.push_back(crtTime);
70   -
  84 + if (_container._size < _nPoints) {
  85 +
  86 + _container._values.push_back(inputElt);
  87 + _container._times.push_back(crtTime);
  88 +
71 89 if (!isNAN(inputElt)) {
72 90 _container._sum += inputElt;
73 91 _container._sumsq += inputElt*inputElt;
74 92 _container._size += 1;
75 93 }
76   - }
77   - _container._ave = (ElemType) _container._sum/_container._size;
78   - _container._sig = std::sqrt((_container._sumsq- _container._sum*_container._sum/_container._size)/_container._size);
79   -
80   - // filter Elements
81   - for(ElemType elt : _container._values){
82   - if(std::abs(elt-_container._ave) > _factor*_container._sig){
83   - elt << NotANumber();
84   - _container._sum -= elt;
85   - _container._sumsq -= elt*elt;
  94 + } else {
  95 + _container._ave = (ElemType) _container._sum / _container._size;
  96 + _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
  97 +
  98 + // filter Elements
  99 + for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
  100 + ElemType crt_val = *it;
  101 + if(isNAN(crt_val))
  102 + continue;
  103 + if (std::abs(crt_val - _container._ave) > _factor * _container._sig) {
  104 + _container._sum -= crt_val;
  105 + _container._sumsq -= crt_val*crt_val;
  106 + _container._size -= 1;
  107 + _container._nFilteredPoints += 1;
  108 + *it << NotANumber();
  109 + }
  110 + }
  111 + _paramOutput->pushTime(_container._times.front());
  112 + _paramOutput->getDataList().push_back(_container._values.front());
  113 + _container._values.pop_front();
  114 + _container._times.pop_front();
  115 + if(!isNAN(_container._values.front())){
86 116 _container._size -= 1;
87   - }
88   - }
89   - _paramOutput->pushTime(_container._times.front());
90   - _paramOutput->getDataList().push_back(_container._values.front());
91   - _container._values.pop_front();
92   - _container._times.pop_front();
93   - _container._size -= 1;
94   -
95   - if(_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess-1){
96   - std::list<double>::iterator itTimes = _container._times.begin();
97   - for(ElemType elt : _container._values) {
98   - _paramOutput->pushTime(*itTimes);
99   - _paramOutput->getDataList().push_back(elt);
100   - ++ itTimes;
101   - }
102   - }
  117 + _container._sum -= _container._values.front();
  118 + _container._sumsq -= _container._values.front() * _container._values.front();
  119 + }
  120 +
  121 + _container._values.push_back(inputElt);
  122 + _container._times.push_back(crtTime);
  123 + if (!isNAN(inputElt)) {
  124 + _container._sum += inputElt;
  125 + _container._sumsq += inputElt*inputElt;
  126 + _container._size += 1;
  127 + }
  128 + }
  129 + // last value filter and write
  130 + if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1) {
  131 + _container._ave = (ElemType) _container._sum / _container._size;
  132 + _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
  133 + for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
  134 + ElemType crt_val = *it;
  135 + if (std::abs(crt_val - _container._ave) > _factor * _container._sig){
  136 + *it << NotANumber();
  137 + _container._nFilteredPoints += 1;
  138 + }
  139 + }
  140 +
  141 + std::list<double>::iterator itTimes = _container._times.begin();
  142 + for (ElemType elt : _container._values) {
  143 + _paramOutput->pushTime(*itTimes);
  144 + _paramOutput->getDataList().push_back(elt);
  145 + ++itTimes;
  146 + }
  147 + }
  148 +
103 149 }
104 150 }
105 151  
... ... @@ -110,14 +156,71 @@ namespace AMDA {
110 156  
111 157 ParamDataSpec<ElemType>* _paramOutput;
112 158  
113   - int _factor;
  159 + double _factor;
114 160 int _nPoints;
115 161 FilteringContainer<ElemType> _container;
116 162  
117 163 };
  164 +
  165 + template <typename ElemType>
  166 + class DataFiltering1D : public Operation {
  167 + public:
  168 + // constructor
  169 +
  170 + DataFiltering1D(Process& pProcess, ParamDataSpec<std::vector<ElemType> >& paramInput, double factor = Default::_factor, int nPoints = Default::_nPoints) :
  171 + Operation(pProcess),
  172 + _paramInput(paramInput),
  173 + _paramOutput(new ParamDataSpec<std::vector<ElemType> >()),
  174 + _factor(factor),
  175 + _nPoints(nPoints) {
  176 + _paramDataOutput = _paramOutput;
  177 + }
  178 +
  179 + virtual ~DataFiltering1D() {
  180 +
  181 + }
  182 +
  183 + void write(ParamDataIndexInfo &pParamDataIndexInfo) {
  184 + // init container
  185 + std::vector<std::list<ElemType> > outputList;
  186 + int size = _paramInput.get(pParamDataIndexInfo._startIndex).size();
  187 + std::vector<int> vecInt(size, 0);
  188 + std::vector<ElemType> vecElement(size, 0);
  189 + _container._size = vecInt;
  190 + _container._nFilteredPoints = vecInt;
  191 + _container._ave = vecElement;
  192 + _container._sum = vecElement;
  193 + _container._sumsq = vecElement;
  194 + _container._sig = vecElement;
  195 + for (int j = 0; j < size; ++j) {
  196 + std::list<ElemType> list;
  197 + _container._values.push_back(list);
  198 + outputList.push_back(list);
  199 + }
  200 + for (int i = 0; i < size; ++i) {
  201 +
  202 + }
  203 +
  204 +
  205 +
  206 +
  207 + }
  208 +
  209 +
  210 +
  211 + private:
  212 + ParamDataSpec<std::vector<ElemType> >& _paramInput;
  213 +
  214 + ParamDataSpec<std::vector<ElemType> >* _paramOutput;
  215 +
  216 + double _factor;
  217 + int _nPoints;
  218 + FilteringContainer1D<ElemType> _container;
  219 +
  220 + };
  221 + }
118 222 }
119 223 }
120   -}
121 224  
122 225 #endif /* DATAFILTERING_HH */
123 226  
... ...
src/ExternLib/DataFiltering/DataFilteringCreator.hh
  Diff comments   View file @9aea9f9
... ... @@ -25,7 +25,7 @@ namespace AMDA {
25 25 class DataFilteringCreator : public VisitorOfParamData {
26 26 public:
27 27  
28   - DataFilteringCreator(Process& pProcess, ParamData& paramInput, int factor=Default::_factor, int nPoints=Default::_nPoints) :
  28 + DataFilteringCreator(Process& pProcess, ParamData& paramInput, double factor=Default::_factor, int nPoints=Default::_nPoints) :
29 29 _process(pProcess),
30 30 _paramData(paramInput),
31 31 _factor(factor),
... ... @@ -173,12 +173,12 @@ namespace AMDA {
173 173 }
174 174 template <typename Type>
175 175 void create1DOperation() {
176   - _operation = new DataFiltering::DataFiltering<Type>(_process, dynamic_cast<ParamDataSpec<Type> &> (_paramData), _factor, _nPoints);
  176 + _operation = new DataFiltering::DataFiltering1D<Type>(_process, dynamic_cast<ParamDataSpec<std::vector<Type>>&> (_paramData), _factor, _nPoints);
177 177 }
178 178  
179 179 Process &_process;
180 180 ParamData &_paramData;
181   - int _factor;
  181 + double _factor;
182 182 int _nPoints;
183 183 Operation* _operation;
184 184 };
... ...
src/ExternLib/DataFiltering/DataFilteringProcess.cc
  Diff comments   View file @9aea9f9
... ... @@ -45,10 +45,10 @@ namespace AMDA {
45 45 TimeStamp DataFilteringProcess::init() {
46 46 TimeStamp timeStamp = _parameterInput->init(this, _timeIntervalList);
47 47 _paramInput = _parameterInput->getParamData(this).get();
48   - int factor = Default::_factor;
  48 + double factor = Default::_factor;
49 49 int nPoints = Default::_nPoints;
50   - if (_attributList.size() == 1)
51   - factor = atoi(_attributList[0].c_str());
  50 + if (_attributList.size() >= 1)
  51 + factor = std::stod(_attributList[0].c_str());
52 52 if (_attributList.size() == 2)
53 53 nPoints = atoi(_attributList[1].c_str());
54 54 DataFilteringCreator ICreator(*this, *_paramInput, factor, nPoints);
... ...
src/ExternLib/DataFiltering/filteringConfig.hh
  Diff comments   View file @9aea9f9
... ... @@ -31,7 +31,7 @@ namespace AMDA {
31 31 }
32 32 namespace Strong{
33 33  
34   - extern const int _factor=4;
  34 + extern const double _factor=4.0;
35 35 extern const int _nPoints=100;
36 36  
37 37 }
... ...