CDPP / AMDA_Kernel

Commit 1aae861e1cb768926cc8df69e566d36f096c737c

Authored by Hacene SI HADJ MOHAND
1 parent 5adfc136
Exists in master and in 69 other branches 10038, 10098, 5390, 5770, 7616, 7875, 8459, 8528, 9016, 9016Bis, 9017, 9017-new, 9049, 9084, 9259, 9326, 9347, 9383, 9719, 9719.new, AllLegends, CDF_ISTP, CDF_ISTP_HOPEFULLY_ALL_WORKING, FER_10400, FER_10412, FER_10663, FER_10765, FER_11000, FER_11100, FER_11242, FER_11277, FER_11500, FER_11537, FER_11588, FER_11651, FER_19836, FER_5344, FER_6031, FER_6598, FER_6966, FER_7318, FER_7618, FER_7857, FER_8455, FER_9629, FER_CR41, FER_CR42, FER_CR43, FER_TFCat, Histo1D2D, JUNO_JEDI, MAZ_10936, MAZ_6018, MAZ_9016_New, MAZ_FIX_9016, SpeasyGet, amdadev, amdadev_test, cu2022, das2, density_maps, develop, evol_9042, feature/9259, pickle, rm_6912, rm_8932, test-merge, ubuntu

compile ok mais segmentations

Showing 1 changed file with 185 additions and 52 deletions   Show diff stats
Inline Side-by-side
src/ExternLib/DataFiltering/DataFiltering.hh
  Diff comments   View file @1aae861
... ... @@ -16,6 +16,7 @@
16 16  
17 17 #include <list>
18 18 #include <c++/4.8.2/bits/stl_vector.h>
  19 +#include <algorithm>
19 20  
20 21 #include "Parameter.hh"
21 22 #include "ParamData.hh"
... ... @@ -26,10 +27,15 @@
26 27 namespace AMDA {
27 28 namespace Parameters {
28 29  
  30 + /**
  31 + * Spike filtering using
  32 + * An algorithm for finding spurious points in turbulent signals, Computers in Physics, V. 7 No. 5, pp. 599 - 607
  33 + */
29 34 namespace DataFiltering {
30 35  
31 36 template <typename ElemType>
32 37 struct FilteringContainer {
  38 + ~FilteringContainer(){}
33 39 // size of real points(not nan and not spike)
34 40 int _size = 0;
35 41 int _nFilteredPoints = 0;
... ... @@ -43,6 +49,10 @@ namespace AMDA {
43 49  
44 50 template <typename ElemType>
45 51 struct FilteringContainer1D {
  52 + FilteringContainer1D(){}
  53 + ~FilteringContainer1D(){
  54 + delete &_size;
  55 + }
46 56 // size of real points(not nan and not spike)
47 57 std::vector<int> _size;
48 58 std::vector<int> _nFilteredPoints;
... ... @@ -50,7 +60,7 @@ namespace AMDA {
50 60 std::vector<ElemType> _sumsq;
51 61 std::vector<ElemType> _ave;
52 62 std::vector<ElemType> _sig;
53   - std::vector<std::list<ElemType> > _values;
  63 + std::list<std::vector<ElemType> > _values;
54 64 std::list<double> _times;
55 65 };
56 66  
... ... @@ -80,7 +90,7 @@ namespace AMDA {
80 90 double crtTime = _paramInput.getTime(_index);
81 91 ElemType inputElt = _paramInput.get(_index);
82 92  
83   - // initialiser et remplir container
  93 + // initialiser et remplir le container container
84 94 if (_container._size < _nPoints) {
85 95  
86 96 _container._values.push_back(inputElt);
... ... @@ -92,72 +102,90 @@ namespace AMDA {
92 102 _container._size += 1;
93 103 }
94 104 } else {
  105 + // traitement une fois le container rempli
95 106 _container._ave = (ElemType) _container._sum / _container._size;
96 107 _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
97 108  
98 109 // filter Elements
99 110 for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
100 111 ElemType crt_val = *it;
101   - if(isNAN(crt_val))
  112 + if (isNAN(crt_val))
102 113 continue;
103 114 if (std::abs(crt_val - _container._ave) > _factor * _container._sig) {
104 115 _container._sum -= crt_val;
105 116 _container._sumsq -= crt_val*crt_val;
106 117 _container._size -= 1;
107 118 _container._nFilteredPoints += 1;
108   - *it << NotANumber();
  119 + *it << NotANumber();
109 120 }
110 121 }
111 122 _paramOutput->pushTime(_container._times.front());
112 123 _paramOutput->getDataList().push_back(_container._values.front());
113 124 _container._values.pop_front();
114 125 _container._times.pop_front();
115   - if(!isNAN(_container._values.front())){
116   - _container._size -= 1;
117   - _container._sum -= _container._values.front();
118   - _container._sumsq -= _container._values.front() * _container._values.front();
  126 + if (!isNAN(_container._values.front())) {
  127 + _container._size -= 1;
  128 + _container._sum -= _container._values.front();
  129 + _container._sumsq -= _container._values.front() * _container._values.front();
119 130 }
120   -
  131 +
121 132 _container._values.push_back(inputElt);
122 133 _container._times.push_back(crtTime);
123   - if (!isNAN(inputElt)) {
  134 + if (!isNAN(inputElt)) {
124 135 _container._sum += inputElt;
125 136 _container._sumsq += inputElt*inputElt;
126 137 _container._size += 1;
127 138 }
128   - }
129   - // last value filter and write
130   - if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1) {
  139 + }
  140 + // last value filter and write
  141 + if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1) {
131 142 _container._ave = (ElemType) _container._sum / _container._size;
132 143 _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
133   - for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
  144 + for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
134 145 ElemType crt_val = *it;
135   - if (std::abs(crt_val - _container._ave) > _factor * _container._sig){
  146 + if (std::abs(crt_val - _container._ave) > _factor * _container._sig) {
136 147 *it << NotANumber();
137 148 _container._nFilteredPoints += 1;
138 149 }
139   - }
140   -
141   - std::list<double>::iterator itTimes = _container._times.begin();
142   - for (ElemType elt : _container._values) {
143   - _paramOutput->pushTime(*itTimes);
144   - _paramOutput->getDataList().push_back(elt);
145   - ++itTimes;
146   - }
147 150 }
148   -
  151 +
  152 + std::list<double>::iterator itTimes = _container._times.begin();
  153 + for (ElemType elt : _container._values) {
  154 + _paramOutput->pushTime(*itTimes);
  155 + _paramOutput->getDataList().push_back(elt);
  156 + ++itTimes;
  157 + }
  158 + }
  159 +
149 160 }
150 161 }
151 162  
152 163  
153 164  
154 165 private:
155   - ParamDataSpec<ElemType>& _paramInput;
156   -
157   - ParamDataSpec<ElemType>* _paramOutput;
158   -
  166 + /**
  167 + * input data
  168 + */
  169 + ParamDataSpec<ElemType >& _paramInput;
  170 +
  171 + /**
  172 + * oupout filtered data
  173 + */
  174 + ParamDataSpec<ElemType >* _paramOutput;
  175 +
  176 + /**
  177 + * factor in term of sigma used in the filtering
  178 + */
159 179 double _factor;
  180 +
  181 + /**
  182 + Number of point used in the fitering algorithm
  183 + */
160 184 int _nPoints;
  185 +
  186 + /**
  187 + * Data container used during filtering
  188 + */
161 189 FilteringContainer<ElemType> _container;
162 190  
163 191 };
... ... @@ -172,50 +200,155 @@ namespace AMDA {
172 200 _paramInput(paramInput),
173 201 _paramOutput(new ParamDataSpec<std::vector<ElemType> >()),
174 202 _factor(factor),
175   - _nPoints(nPoints) {
  203 + _nPoints(nPoints),
  204 + _container(new FilteringContainer1D<ElemType> ()){
176 205 _paramDataOutput = _paramOutput;
177 206 }
178 207  
179 208 virtual ~DataFiltering1D() {
180   -
181 209 }
182 210  
183 211 void write(ParamDataIndexInfo &pParamDataIndexInfo) {
  212 +
184 213 // init container
185   - std::vector<std::list<ElemType> > outputList;
  214 +
186 215 int size = _paramInput.get(pParamDataIndexInfo._startIndex).size();
187   - std::vector<int> vecInt(size, 0);
188   - std::vector<ElemType> vecElement(size, 0);
189   - _container._size = vecInt;
190   - _container._nFilteredPoints = vecInt;
191   - _container._ave = vecElement;
192   - _container._sum = vecElement;
193   - _container._sumsq = vecElement;
194   - _container._sig = vecElement;
195   - for (int j = 0; j < size; ++j) {
196   - std::list<ElemType> list;
197   - _container._values.push_back(list);
198   - outputList.push_back(list);
199   - }
200   - for (int i = 0; i < size; ++i) {
  216 + std::vector<int> vectInt(size,0);
  217 + _container->_size = vectInt;
  218 + /**
  219 + _container->_nFilteredPoints.resize(size,0);
  220 + _container->_ave.resize(size,0);
  221 + _container->_sum.resize(size,0);
  222 + _container->_sumsq.resize(size,0);
  223 + _container->_sig.resize(size,0);
  224 + */
  225 + for (unsigned int _index = pParamDataIndexInfo._startIndex;
  226 + _index < pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess;
  227 + ++_index) {
  228 + double crtTime = _paramInput.getTime(_index);
  229 + std::vector<ElemType> inputElt = _paramInput.get(_index);
201 230  
202   - }
203   -
204   -
205   -
206   -
207   - }
  231 + _paramOutput->pushTime(crtTime);
  232 + _paramOutput->getDataList().push_back(inputElt);
  233 + /**
208 234  
  235 + // initialiser et remplir container
  236 + if (*std::min_element(std::begin(_container._size), std::end(_container._size)) < _nPoints) {
  237 +
  238 + _container._values.push_back(inputElt);
  239 + _container._times.push_back(crtTime);
  240 + for (int i = 0; i < size; i++) {
  241 + if (!isNAN(inputElt[i])) {
  242 + _container._sum[i] += inputElt[i];
  243 + _container._sumsq[i] += inputElt[i]*inputElt[i];
  244 + _container._size[i] += 1;
  245 + }
  246 + }
  247 + } else {
  248 + for (int i = 0; i < size; i++) {
  249 + _container._ave[i] = (ElemType) _container._sum[i] / _container._size[i];
  250 + _container._sig[i] = std::sqrt((_container._sumsq[i] - _container._sum[i] * _container._sum[i] / _container._size[i]) / _container._size[i]);
  251 + }
  252 +
  253 + // filter Elements
  254 + for (auto elt = _container._values.begin(); elt != _container._values.end(); ++elt) {
  255 + int i;
  256 + for (auto it = elt->begin(); it != elt->end(); ++it) {
  257 + i = it - elt->begin();
  258 + ElemType crt_val = *it;
  259 + if (isNAN(crt_val)) {
  260 + continue;
  261 + }
  262 +
  263 + if (std::abs(crt_val - _container._ave[i]) > _factor * _container._sig[i]) {
  264 + _container._sum[i] -= crt_val;
  265 + _container._sumsq[i] -= crt_val*crt_val;
  266 + _container._size[i] -= 1;
  267 + _container._nFilteredPoints[i] += 1;
  268 + *it << NotANumber();
  269 + }
  270 + }
  271 + }
  272 + _paramOutput->pushTime(_container._times.front());
  273 + _paramOutput->getDataList().push_back(_container._values.front());
  274 + _container._values.pop_front();
  275 + _container._times.pop_front();
  276 + for (int i = 0; i < size; i++) {
  277 + if (!isNAN(_container._values.front()[i])) {
  278 + _container._size[i] -= 1;
  279 + _container._sum[i] -= _container._values.front()[i];
  280 + _container._sumsq[i] -= _container._values.front()[i] * _container._values.front()[i];
  281 + }
  282 + if (!isNAN(inputElt[i])) {
  283 + _container._sum[i] += inputElt[i];
  284 + _container._sumsq[i] += inputElt[i] * inputElt[i];
  285 + _container._size[i] += 1;
  286 + }
  287 + }
  288 + _container._values.push_back(inputElt);
  289 + _container._times.push_back(crtTime);
  290 + }
  291 + // last value filter and write
  292 + if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1) {
  293 + for (int i = 0; i < size; i++) {
  294 + _container._ave[i] = (ElemType) _container._sum[i] / _container._size[i];
  295 + _container._sig[i] = std::sqrt((_container._sumsq[i] - _container._sum[i] * _container._sum[i] / _container._size[i]) / _container._size[i]);
  296 + }
  297 + for (auto elt = _container._values.begin(); elt != _container._values.end(); ++elt) {
  298 + int i;
  299 + for (auto it = elt->begin(); it != elt->end(); ++it) {
  300 + i = it - elt->begin();
  301 + ElemType crt_val = *it;
  302 + if (isNAN(crt_val)) {
  303 + continue;
  304 + }
  305 +
  306 + if (std::abs(crt_val - _container._ave[i]) > _factor * _container._sig[i]) {
  307 + _container._sum[i] -= crt_val;
  308 + _container._sumsq[i] -= crt_val*crt_val;
  309 + _container._size[i] -= 1;
  310 + _container._nFilteredPoints[i] += 1;
  311 + *it << NotANumber();
  312 + }
  313 + }
  314 + }
209 315  
  316 + std::list<double>::iterator itTimes = _container._times.begin();
  317 + for (auto elt : _container._values) {
  318 + _paramOutput->pushTime(*itTimes);
  319 + _paramOutput->getDataList().push_back(elt);
  320 + ++itTimes;
  321 + }
  322 + }
  323 + * */
  324 + }
  325 + }
210 326  
211 327 private:
  328 + /**
  329 + * input data
  330 + */
212 331 ParamDataSpec<std::vector<ElemType> >& _paramInput;
213 332  
  333 + /**
  334 + * oupout filtered data
  335 + */
214 336 ParamDataSpec<std::vector<ElemType> >* _paramOutput;
215 337  
  338 + /**
  339 + * factor in term of sigma used in the filtering
  340 + */
216 341 double _factor;
  342 +
  343 + /**
  344 + Number of point used in the fitering algorithm
  345 + */
217 346 int _nPoints;
218   - FilteringContainer1D<ElemType> _container;
  347 +
  348 + /**
  349 + * Data container used during filtering
  350 + */
  351 + FilteringContainer1D<ElemType>* _container;
219 352  
220 353 };
221 354 }
... ...