CDPP / HELIOPROPA

Commit a2c1b3ca8b584af42beff2066c811eba0992f354

Authored by Goutte
1 parent 3309d7c3

Fix an issue with data being ignored. Works on Mercury. 

Uranus and Neptune still lack data.
Showing 1 changed file with 68 additions and 58 deletions   Show diff stats
Inline Side-by-side
@@ -704,13 +704,14 @@ def get_data_for_target(target_config, input_slug, @@ -704,13 +704,14 @@ def get_data_for_target(target_config, input_slug,
704 except Exception: 704 except Exception:
705 log.error("Failed to parse time from %s." % ltime) 705 log.error("Failed to parse time from %s." % ltime)
706 raise 706 raise
  707 + # Keep only what's in the interval
707 if s0 <= dtime <= s1: 708 if s0 <= dtime <= s1:
708 dkey = round_time(dtime, 60*60).strftime(precision) 709 dkey = round_time(dtime, 60*60).strftime(precision)
709 orbit_data[dkey] = datum_hee 710 orbit_data[dkey] = datum_hee
710 cdf_handle.close() 711 cdf_handle.close()
711 712
712 all_data = {} # keys are datetime as str, values tuples of data 713 all_data = {} # keys are datetime as str, values tuples of data
713 - strip_before_date = None 714 +
714 for model in models: 715 for model in models:
715 s0, s1 = _sta_sto(model, started_at, stopped_at) 716 s0, s1 = _sta_sto(model, started_at, stopped_at)
716 model_files = retrieve_amda_netcdf( 717 model_files = retrieve_amda_netcdf(
@@ -748,6 +749,12 @@ def get_data_for_target(target_config, input_slug, @@ -748,6 +749,12 @@ def get_data_for_target(target_config, input_slug,
748 # Alpha, RamP, E, Beta, Ma, Kp, R, DST, 749 # Alpha, RamP, E, Beta, Ma, Kp, R, DST,
749 # AE, Flux, Flag, F10_Index, StartTime, StopTime 750 # AE, Flux, Flag, F10_Index, StartTime, StopTime
750 751
  752 + # Don't ignore, but instead, compute a mean ?
  753 + # Look for discretisation ? temporal drizzling ? 1D drizzling ?
  754 + # The easy linear mean feels a bit rough too. Running mean too.
  755 + # FIXME
  756 + ignored_count = 0
  757 +
751 log.debug("%s: aggregating data from '%s'..." % 758 log.debug("%s: aggregating data from '%s'..." %
752 (target_config['name'], model_file)) 759 (target_config['name'], model_file))
753 for ltime, datum_v, datum_b, datum_t, datum_n, datum_p, datum_a \ 760 for ltime, datum_v, datum_b, datum_t, datum_n, datum_p, datum_a \
@@ -759,63 +766,66 @@ def get_data_for_target(target_config, input_slug, @@ -759,63 +766,66 @@ def get_data_for_target(target_config, input_slug,
759 log.error("Failed to parse time from %s." % ltime) 766 log.error("Failed to parse time from %s." % ltime)
760 raise 767 raise
761 768
762 - if s0 <= dtime <= s1:  
763 - droundtime = round_time(dtime, 60*60)  
764 - dkey = droundtime.strftime(precision)  
765 -  
766 - x_hee = None  
767 - y_hee = None  
768 - if dkey in orbit_data:  
769 - x_hee = orbit_data[dkey][0] / ASTRONOMICAL_UNIT_IN_KM  
770 - y_hee = orbit_data[dkey][1] / ASTRONOMICAL_UNIT_IN_KM  
771 -  
772 - # First exception: V may be a vector instead of a scalar  
773 - if hasattr(datum_v, '__len__'):  
774 - vrad = datum_v[0]  
775 - vtan = datum_v[1]  
776 - vtot = sqrt(vrad * vrad + vtan * vtan)  
777 - else: # eg: Earth  
778 - vrad = None  
779 - vtan = None  
780 - vtot = datum_v  
781 -  
782 - # Second exception: Earth is always at (1, 0)  
783 - if target_config['slug'] == 'earth':  
784 - x_hee = 1  
785 - y_hee = 0  
786 -  
787 - # Third exception: B is a Vector3 or a Vector2 for Earth  
788 - if target_config['slug'] == 'earth':  
789 - if model['slug'] == 'omni_hour_all': # Vector3  
790 - datum_b = datum_b[0]  
791 - # if model['slug'] == 'ace_swepam_real': # Vector2  
792 - # datum_b = datum_b[0]  
793 - if model['slug'] == 'omni_hour_all':  
794 - datum_p = datum_n * vtot * vtot * 1.6726e-6  
795 - if model['slug'] == 'ace_swepam_real':  
796 - datum_p = datum_n * vtot * vtot * 1.6726e-6  
797 - if vtot is None or isnan(vtot):  
798 - continue  
799 -  
800 - # Keep adding exceptions here until you can't or become mad  
801 -  
802 - first = False  
803 - if strip_before_date is None:  
804 - first = True  
805 - strip_before_date = droundtime  
806 -  
807 - # First model has priority, and also sets a leftmost time  
808 - if first or (  
809 - (dkey not in all_data) and  
810 - (droundtime > strip_before_date)  
811 - ):  
812 - # /!\ MUST be in the same order as PROPERTIES  
813 - all_data[dkey] = (  
814 - dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),  
815 - vrad, vtan, vtot,  
816 - datum_b, datum_t, datum_p, datum_n, datum_a,  
817 - x_hee, y_hee  
818 - ) 769 + if not (s0 <= dtime <= s1):
  770 + continue # Cull what's out of the interval
  771 +
  772 + droundtime = round_time(dtime, 60*60)
  773 + dkey = droundtime.strftime(precision)
  774 +
  775 + x_hee = None
  776 + y_hee = None
  777 + if dkey in orbit_data:
  778 + x_hee = orbit_data[dkey][0] / ASTRONOMICAL_UNIT_IN_KM
  779 + y_hee = orbit_data[dkey][1] / ASTRONOMICAL_UNIT_IN_KM
  780 +
  781 + # First exception: V may be a vector instead of a scalar
  782 + if hasattr(datum_v, '__len__'):
  783 + vrad = datum_v[0]
  784 + vtan = datum_v[1]
  785 + vtot = sqrt(vrad * vrad + vtan * vtan)
  786 + else: # eg: Earth
  787 + vrad = None
  788 + vtan = None
  789 + vtot = datum_v
  790 +
  791 + # Second exception: Earth is always at (1, 0)
  792 + if target_config['slug'] == 'earth':
  793 + x_hee = 1
  794 + y_hee = 0
  795 +
  796 + # Third exception: B is a Vector3 or a Vector2 for Earth
  797 + if target_config['slug'] == 'earth':
  798 + if model['slug'] == 'omni_hour_all': # Vector3
  799 + datum_b = datum_b[0]
  800 + # if model['slug'] == 'ace_swepam_real': # Vector2
  801 + # datum_b = datum_b[0]
  802 + if model['slug'] == 'omni_hour_all':
  803 + datum_p = datum_n * vtot * vtot * 1.6726e-6 # ?
  804 + if model['slug'] == 'ace_swepam_real':
  805 + datum_p = datum_n * vtot * vtot * 1.6726e-6 # ?
  806 + if vtot is None or isnan(vtot):
  807 + continue
  808 +
  809 + # Keep adding exceptions here until you can't or become mad
  810 +
  811 + # Crude/Bad drizzling condition : first found datum in the hour
  812 + # gets to set the data. Improve this.
  813 + if dkey not in all_data:
  814 + # /!. The Set value MUST be in the same order as PROPERTIES
  815 + all_data[dkey] = (
  816 + dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
  817 + vrad, vtan, vtot,
  818 + datum_b, datum_t, datum_p, datum_n, datum_a,
  819 + x_hee, y_hee
  820 + )
  821 + else:
  822 + ignored_count += 1
  823 +
  824 + # Improve this loop so as to remove this stinky debug log
  825 + if ignored_count > 0:
  826 + log.debug(" Ignored %d datum(s) during ~\"drizzling\"."
  827 + % ignored_count)
  828 +
819 cdf_handle.close() 829 cdf_handle.close()
820 830
821 return all_data 831 return all_data