CDPP / AMDA_Kernel

Commit bcc46bb64747fa6e141003fc0fbf2cdd950ee609

Authored by Hacene SI HADJ MOHAND
1 parent 7320705f
Exists in master and in 67 other branches 10038, 10098, 5390, 5770, 7616, 7875, 8459, 8528, 9016, 9016Bis, 9017, 9017-new, 9049, 9084, 9259, 9326, 9347, 9383, 9719, 9719.new, AllLegends, CDF_ISTP, CDF_ISTP_HOPEFULLY_ALL_WORKING, FER_10400, FER_10412, FER_10663, FER_10765, FER_11000, FER_11100, FER_11242, FER_11277, FER_11500, FER_11537, FER_11588, FER_11651, FER_19836, FER_5344, FER_6031, FER_6598, FER_6966, FER_7318, FER_7618, FER_7857, FER_8455, FER_9629, FER_CR41, FER_CR42, FER_CR43, FER_TFCat, Histo1D2D, JUNO_JEDI, MAZ_10936, MAZ_6018, MAZ_9016_New, MAZ_FIX_9016, SpeasyGet, amdadev, amdadev_test, cu2022, das2, density_maps, develop, evol_9042, feature/9259, pickle, test-merge, ubuntu

correcting spikes romval algorithm + log file

Showing 1 changed file with 102 additions and 47 deletions   Show diff stats
Inline Side-by-side
src/ExternLib/DataFiltering/DataFiltering.hh
  Diff comments   View file @bcc46bb
... ... @@ -134,16 +134,18 @@ namespace AMDA {
134 134 }
135 135  
136 136 void write(ParamDataIndexInfo &pParamDataIndexInfo) {
137   - std::fstream fs;
138   - fs.open("dataFiltering0D.log", std::fstream::in | std::fstream::out | std::fstream::app);
139   - if(pParamDataIndexInfo._startIndex ==0 && pParamDataIndexInfo._nbDataToProcess >0){
140   - std::chrono::system_clock::time_point today = std::chrono::system_clock::now();
141   - std::time_t tt;
142   - tt = std::chrono::system_clock::to_time_t ( today );
143   - fs <<"# "<< std::ctime(&tt)<<"# [Debug] outPut of dataFiltering plugin"<<std::endl;
144   - fs <<"# time"<<" "<<"removed_value"<<" average"<<" "<<"std"<<" "<<"factor"<<std::endl;
145   - }
146   -
  137 + std::fstream fs;
  138 + fs.open("dataFiltering0D.log", std::fstream::in | std::fstream::out | std::fstream::app);
  139 + if (pParamDataIndexInfo._startIndex == 0 && pParamDataIndexInfo._nbDataToProcess > 0) {
  140 + std::chrono::system_clock::time_point today = std::chrono::system_clock::now();
  141 + std::time_t tt;
  142 + tt = std::chrono::system_clock::to_time_t(today);
  143 + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering0D plugin" << std::endl;
  144 + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl;
  145 + fs << "# time" << " " << "removed_value" << " average" << " " << "std" << " " << "factor" << std::endl;
  146 + }
  147 + bool isFirstSet = true;
  148 +
147 149 for (unsigned int _index = pParamDataIndexInfo._startIndex;
148 150 _index < pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess;
149 151 ++_index) {
... ... @@ -165,17 +167,21 @@ namespace AMDA {
165 167 // traitement une fois le container rempli
166 168 _container._ave = (ElemType) _container._sum / _container._size;
167 169 _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
168   -
  170 + int offSet = _nPoints / 4;
  171 + if (isFirstSet)
  172 + offSet = 0;
169 173 // filter Elements
170   - std::list<double>::iterator itTime = _container._times.begin();
171   - for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
  174 + std::list<double>::iterator itTime = std::next(_container._times.begin(), offSet);
  175 + auto startVal = std::next(_container._values.begin(), offSet);
  176 + auto endVal = std::next(_container._values.begin(), 2 * offSet);
  177 + for (auto it = startVal; it != endVal; ++it) {
172 178 ElemType crt_val = *it;
173 179 if (isNAN(crt_val))
174 180 continue;
175 181 if (std::abs(crt_val - _container._ave) > _factor * _container._sig) {
176   - fs<<TimeUtil::formatTimeDateInIso(*itTime);
177   - fs <<" "<< std::to_string(crt_val) <<" "<<_container._ave<<" "<<_container._sig;
178   - fs<<" "<<_factor<<std::endl;
  182 + fs << TimeUtil::formatTimeDateInIso(*itTime);
  183 + fs << " " << std::to_string(crt_val) << " " << _container._ave << " " << _container._sig;
  184 + fs << " " << _factor << std::endl;
179 185 _container._sum -= crt_val;
180 186 _container._sumsq -= crt_val*crt_val;
181 187 _container._size -= 1;
... ... @@ -201,37 +207,38 @@ namespace AMDA {
201 207 _container._sumsq += inputElt*inputElt;
202 208 _container._size += 1;
203 209 }
  210 + isFirstSet = false;
204 211 }
205 212 // last value filter and write
206 213 if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1 && !_container._values.empty()) {
207 214 _container._ave = (ElemType) _container._sum / _container._size;
208 215 _container._sig = std::sqrt((_container._sumsq - _container._sum * _container._sum / _container._size) / _container._size);
209   - std::list<double>::iterator itTime = _container._times.begin();
  216 + std::list<double>::iterator itTime = _container._times.begin();
210 217 for (auto it = _container._values.begin(); it != _container._values.end(); ++it) {
211 218 ElemType crt_val = *it;
212 219 if (std::abs(crt_val - _container._ave) > _factor * _container._sig) {
213   - fs<<TimeUtil::formatTimeDateInIso(*itTime);
214   - fs << std::to_string(crt_val) <<" "<<_container._ave<<" "<<_container._sig;
215   - fs<<" "<<_factor<<std::endl;
  220 + fs << TimeUtil::formatTimeDateInIso(*itTime);
  221 + fs <<" "<< std::to_string(crt_val) << " " << _container._ave << " " << _container._sig;
  222 + fs << " " << _factor << std::endl;
216 223 *it << NotANumber();
217   - _container._nFilteredPoints += 1;
  224 + _container._nFilteredPoints += 1;
218 225 }
219 226 _paramOutput->pushTime(*itTime);
220 227 _paramOutput->getDataList().push_back(crt_val);
221   - ++ itTime;
  228 + ++itTime;
222 229 }
223 230 }
224 231  
225 232 }
226   - // clear container
227   - _container._ave = 0;
228   - _container._sig = 0;
229   - _container._size = 0;
230   - _container._sum =0;
231   - _container._times.clear();
232   - _container._sumsq = 0;
233   - _container._values.clear();
234   - fs.close();
  233 + // clear container
  234 + _container._ave = 0;
  235 + _container._sig = 0;
  236 + _container._size = 0;
  237 + _container._sum = 0;
  238 + _container._times.clear();
  239 + _container._sumsq = 0;
  240 + _container._values.clear();
  241 + fs.close();
235 242 }
236 243  
237 244  
... ... @@ -282,6 +289,17 @@ namespace AMDA {
282 289 }
283 290  
284 291 void write(ParamDataIndexInfo &pParamDataIndexInfo) {
  292 + std::fstream fs;
  293 + fs.open("dataFiltering1D.log", std::fstream::in | std::fstream::out | std::fstream::app);
  294 + if (pParamDataIndexInfo._startIndex == 0 && pParamDataIndexInfo._nbDataToProcess > 0) {
  295 + std::chrono::system_clock::time_point today = std::chrono::system_clock::now();
  296 + std::time_t tt;
  297 + tt = std::chrono::system_clock::to_time_t(today);
  298 + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering1D plugin" << std::endl;
  299 + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl;
  300 + fs << "#index" << " " << "time" << " " << "removed_value" << " average" << " " << "std" << " " << "factor" << std::endl;
  301 + }
  302 + bool isFirstSet = true;
285 303 int size = 0;
286 304 for (unsigned int _index = pParamDataIndexInfo._startIndex;
287 305 _index < pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess;
... ... @@ -290,7 +308,7 @@ namespace AMDA {
290 308 std::vector<ElemType> inputElt = _paramInput.get(_index);
291 309  
292 310 if (_container->_values.empty()) {
293   - size = inputElt.size();
  311 + size = inputElt.size();
294 312 _container->init(size);
295 313 }
296 314 // initialiser et remplir container
... ... @@ -312,7 +330,14 @@ namespace AMDA {
312 330 }
313 331  
314 332 // filter Elements
315   - for (auto elt = _container->_values.begin(); elt != _container->_values.end(); ++elt) {
  333 + int offSet = _nPoints / 4;
  334 + if (isFirstSet)
  335 + offSet = 0;
  336 + // filter Elements
  337 + std::list<double>::iterator itTime = std::next(_container->_times.begin(), offSet);
  338 + auto startVal = std::next(_container->_values.begin(), offSet);
  339 + auto endVal = std::next(_container->_values.begin(), 2 * offSet);
  340 + for (auto elt = startVal; elt != endVal; ++elt) {
316 341 int i;
317 342 for (auto it = elt->begin(); it != elt->end(); ++it) {
318 343 i = it - elt->begin();
... ... @@ -322,6 +347,9 @@ namespace AMDA {
322 347 }
323 348  
324 349 if (std::abs(crt_val - _container->_ave[i]) > _factor * _container->_sig[i]) {
  350 + fs << i << " " << TimeUtil::formatTimeDateInIso(*itTime);
  351 + fs <<" "<< std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i];
  352 + fs << " " << _factor << std::endl;
325 353 _container->_sum[i] -= crt_val;
326 354 _container->_sumsq[i] -= crt_val*crt_val;
327 355 _container->_size[i] -= 1;
... ... @@ -329,6 +357,7 @@ namespace AMDA {
329 357 *it << NotANumber();
330 358 }
331 359 }
  360 + ++itTime;
332 361 }
333 362 _paramOutput->pushTime(_container->_times.front());
334 363 _paramOutput->getDataList().push_back(_container->_values.front());
... ... @@ -348,6 +377,7 @@ namespace AMDA {
348 377 }
349 378 _container->_values.push_back(inputElt);
350 379 _container->_times.push_back(crtTime);
  380 + isFirstSet = false;
351 381 }
352 382 // last value filter and write
353 383 if (_index == pParamDataIndexInfo._startIndex + pParamDataIndexInfo._nbDataToProcess - 1 && !_container->_values.empty()) {
... ... @@ -355,6 +385,8 @@ namespace AMDA {
355 385 _container->_ave[i] = (ElemType) _container->_sum[i] / _container->_size[i];
356 386 _container->_sig[i] = std::sqrt((_container->_sumsq[i] - _container->_sum[i] * _container->_sum[i] / _container->_size[i]) / _container->_size[i]);
357 387 }
  388 +
  389 + std::list<double>::iterator itTime = _container->_times.begin();
358 390 for (auto elt = _container->_values.begin(); elt != _container->_values.end(); ++elt) {
359 391 int i;
360 392 for (auto it = elt->begin(); it != elt->end(); ++it) {
... ... @@ -365,6 +397,9 @@ namespace AMDA {
365 397 }
366 398  
367 399 if (std::abs(crt_val - _container->_ave[i]) > _factor * _container->_sig[i]) {
  400 + fs << i << " " << TimeUtil::formatTimeDateInIso(*itTime);
  401 + fs << std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i];
  402 + fs << " " << _factor << std::endl;
368 403 _container->_sum[i] -= crt_val;
369 404 _container->_sumsq[i] -= crt_val*crt_val;
370 405 _container->_size[i] -= 1;
... ... @@ -372,17 +407,13 @@ namespace AMDA {
372 407 *it << NotANumber();
373 408 }
374 409 }
375   - }
376   -
377   - std::list<double>::iterator itTimes = _container->_times.begin();
378   - for (auto elt : _container->_values) {
379   - _paramOutput->pushTime(*itTimes);
380   - _paramOutput->getDataList().push_back(elt);
381   - ++itTimes;
  410 + _paramOutput->pushTime(*itTime);
  411 + _paramOutput->getDataList().push_back(*elt);
  412 + ++itTime;
382 413 }
383 414 }
384 415 }
385   -
  416 + fs.close();
386 417 }
387 418 private:
388 419 /**
... ... @@ -429,6 +460,17 @@ namespace AMDA {
429 460 }
430 461  
431 462 void write(ParamDataIndexInfo &pParamDataIndexInfo) {
  463 + std::fstream fs;
  464 + fs.open("dataFiltering1D.log", std::fstream::in | std::fstream::out | std::fstream::app);
  465 + if (pParamDataIndexInfo._startIndex == 0 && pParamDataIndexInfo._nbDataToProcess > 0) {
  466 + std::chrono::system_clock::time_point today = std::chrono::system_clock::now();
  467 + std::time_t tt;
  468 + tt = std::chrono::system_clock::to_time_t(today);
  469 + fs << "# " << std::ctime(&tt) << "# [Debug] outPut of dataFiltering2D plugin" << std::endl;
  470 + fs<<"# Expression/Param: "<<_process.getExpression()<< std::endl;
  471 + fs << "#index1 Index2 time removed_value average std factor" << std::endl;
  472 + }
  473 + bool isFirstSet = true;
432 474 // init container
433 475 int size1 = 0;
434 476 int size2 = 0;
... ... @@ -465,9 +507,15 @@ namespace AMDA {
465 507 _container->_ave[i] = (ElemType) _container->_sum[i] / _container->_size[i];
466 508 _container->_sig[i] = std::sqrt((_container->_sumsq[i] - _container->_sum[i] * _container->_sum[i] / _container->_size[i]) / _container->_size[i]);
467 509 }
468   -
  510 + // filter elements
  511 + int offSet = _nPoints / 4;
  512 + if (isFirstSet)
  513 + offSet = 0;
469 514 // filter Elements
470   - for (auto elt = _container->_2Dvalues.begin(); elt != _container->_2Dvalues.end(); ++elt) {
  515 + std::list<double>::iterator itTime = std::next(_container->_times.begin(), offSet);
  516 + auto startVal = std::next(_container->_2Dvalues.begin(), offSet);
  517 + auto endVal = std::next(_container->_2Dvalues.begin(), 2 * offSet);
  518 + for (auto elt = startVal; elt != endVal; ++elt) {
471 519 Tab2DData<ElemType> outputElt(size1, size2);
472 520 Tab2DData<ElemType> inputElt = *elt;
473 521 for (int i = 0; i < size1; i++) {
... ... @@ -478,6 +526,9 @@ namespace AMDA {
478 526 if (isNAN(crt_val)) {
479 527 outputElt[i][j] = res;
480 528 } else if (std::abs(crt_val - _container->_ave[i * size2 + j]) > _factor * _container->_sig[i * size2 + j]) {
  529 + fs << i << " " << j << " " << TimeUtil::formatTimeDateInIso(*itTime);
  530 + fs <<" "<< std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i];
  531 + fs << " " << _factor << std::endl;
481 532 _container->_sum[i * size2 + j] -= crt_val;
482 533 _container->_sumsq[i * size2 + j] -= crt_val*crt_val;
483 534 _container->_size[i * size2 + j] -= 1;
... ... @@ -513,6 +564,7 @@ namespace AMDA {
513 564 }
514 565 _container->_2Dvalues.push_back(inputElt);
515 566 _container->_times.push_back(crtTime);
  567 + isFirstSet = false;
516 568 }
517 569  
518 570 // last value filter and write
... ... @@ -525,7 +577,7 @@ namespace AMDA {
525 577 // filter values and write
526 578 // filter Elements
527 579  
528   - std::list<double>::iterator itTimes = _container->_times.begin();
  580 + std::list<double>::iterator itTime = _container->_times.begin();
529 581 for (auto elt = _container->_2Dvalues.begin(); elt != _container->_2Dvalues.end(); ++elt) {
530 582 Tab2DData<ElemType> outputElt(size1, size2);
531 583 Tab2DData<ElemType> inputElt = *elt;
... ... @@ -533,6 +585,9 @@ namespace AMDA {
533 585 for (int j = 0; j < size2; j++) {
534 586 ElemType crt_val = inputElt[i][j];
535 587 if (std::abs(crt_val - _container->_ave[i * size2 + j]) > _factor * _container->_sig[i * size2 + j]) {
  588 + fs << i << " " << j << " " << TimeUtil::formatTimeDateInIso(*itTime);
  589 + fs << std::to_string(crt_val) << " " << _container->_ave[i] << " " << _container->_sig[i];
  590 + fs << " " << _factor << std::endl;
536 591 _container->_sum[i * size2 + j] -= crt_val;
537 592 _container->_sumsq[i * size2 + j] -= crt_val*crt_val;
538 593 _container->_size[i * size2 + j] -= 1;
... ... @@ -546,15 +601,15 @@ namespace AMDA {
546 601 }
547 602 // write
548 603 *elt = outputElt;
549   - _paramOutput->pushTime(*itTimes);
  604 + _paramOutput->pushTime(*itTime);
550 605 _paramOutput->getDataList().push_back(outputElt);
551 606 outputElt.free();
552   - ++itTimes;
  607 + ++itTime;
553 608 }
554 609 }
555 610  
556 611 }
557   -
  612 + fs.close();
558 613 }
559 614 private:
560 615 /*
... ...