Enable Earth

Enable more precise configuration of each models' parameters

Enable Earth
Enable more precise configuration of each models' parameters
Goutte
1 parent 952e3d8f
Showing 7 changed files with 138 additions and 59 deletions Show diff stats
CHANGELOG.md
README.md
config.yml
web/run.py
web/static/js/swapp.js
web/static/js/swapp.ls
web/view/home.html.jinja2
@@ -5,13 +5,22 @@
 - [ ] Add a README to the download tarball
 - [ ] Set the log level to _error_ in production (see `web/run.py`)
 - [ ] CRON statements to call the cache cleanup and warmup
-- [ ] Cache warmup (generate for today's default interval) `/cache/warmup`
 - [ ] Give the future data another color
  
 An heliospheric propagation 1D MHD model for solar wind prediction at planets, probes and comets.
  
  
  
+## 1.0.0-rc5
+
+- [ ] Rework the images of Rosetta and Juno
+- [ ] Cache warmup (generate for today's default interval) `/cache/warmup`
+- [ ] Improve the generated CDF for SAMP
+- [ ] Enable p67
+- [x] Enable Earth
+- [x] Enable more precise configuration of each models' parameters
+
+
 ## 1.0.0-rc4
  
 - [x] Sort times series by closeness to the sun
@@ -24,6 +24,7 @@ It also gathers NetCDF data from AMDA, and serves it as CSV to the plotter.
 - `config.yml` : the main configuration file.
 - `web/run.py` : the front controller, holding most of the code.
 - `web/view/home.html.jinja2` : the HTML template.
+- `web/static/js/swapp.ls` : most of the javascript client-side.
  
  
 ## Install
@@ -98,15 +98,17 @@ targets:
     - type: 'planet'
       slug: 'earth'
       name: 'Earth'
-      title: 'Earth (coming soon)'
+      title: 'Earth'
       orbit:
-        models:
-          - slug: 'earth_orb_all'
-        semimajor: 0
-        semiminor: 0
+        models: [] # Earth has no orbit models, we hard-coded it to (1, 0)
       models:
-        - slug: 'tao_earth_sw'
-      locked: true
+        - slug: 'omni_hour_all'
+        - slug: 'ace_swepam_real'
+          parameters:
+            dens: 'Dens'
+            vtot: 'Vel'
+            temp: 'Temp'
+      locked: false
       default: true
     - type: 'planet'
       slug: 'mars'
@@ -159,6 +161,8 @@ targets:
             stopped_at: '2014-08-02T00:00:00'
           - slug: 'p67_orb_all'
             started_at: '2014-08-02T00:00:00'
+            parameters:
+              hee: 'XYZ_HEE'
       models:
           - slug: 'tao_ros_sw'
       locked: false
@@ -184,6 +188,8 @@ targets:
       orbit:
         models:
           - slug: 'p67_orb_all'
+            parameters:
+              hee: 'XYZ_HEE'
       locked: true
       default: false
  
@@ -74,7 +74,7 @@ CACHE_DIR = get_path(&#39;../cache&#39;)
 # innermost loop of `get_data_for_target`.
 # The javascript knows the targets' properties under these names.
 PROPERTIES = ('time', 'vrad', 'vtan', 'vtot', 'btan', 'temp', 'pdyn', 'dens',
-              'angl', 'xhee', 'yhee')
+              'atse', 'xhee', 'yhee')
  
 # The parameters that the users can handle.
 # The slug MUST be one of the properties above.
@@ -119,8 +119,8 @@ PARAMETERS = {
         'active': False,
         'position': 50,
     },
-    'angl': {
-        'slug': 'angl',
+    'atse': {
+        'slug': 'atse',
         'name': 'Angle T-S-E',
         'title': 'Angle Target-Sun-Earth.',
         'units': 'deg',
@@ -426,9 +426,10 @@ def get_data_for_target(target_config, started_at, stopped_at):
               % (target_config['slug'], str(e)))
     try:
         orbits = target_config['orbit']['models']
-    except Exception as e:
-        abort(500, "Invalid orbit configuration for '%s' : %s"
-              % (target_config['slug'], str(e)))
+    except KeyError as e:
+        orbits = []
+        # abort(500, "Invalid orbit configuration for '%s' : %s"
+        #       % (target_config['slug'], str(e)))
  
     def _sta_sto(_cnf, _sta, _sto):
         if 'started_at' in _cnf:
@@ -443,12 +444,36 @@ def get_data_for_target(target_config, started_at, stopped_at):
             _s1 = _sto
         return _s0, _s1
  
+    def _read_var(_nc, _keys, _key, mandatory=False):
+        try:
+            return _nc.variables[_keys[_key]]
+        except KeyError:
+            pass
+        if mandatory:
+            raise Exception("No variable '%s' found in NetCDF." % _keys[_key])
+        return [None] * len(_nc.variables['Time'])  # slow -- use numpy!
+
+    # Override them using the configuration, maybe also put these in config ?
+    default_nc_keys = {
+        'hee': 'HEE',
+        'vtot': 'V',
+        'magn': 'B',
+        'temp': 'T',
+        'dens': 'N',
+        'pdyn': 'P_dyn',
+        'atse': 'Delta_angle',
+    }
+
     precision = "%Y-%m-%dT%H"  # model and orbits times are only equal-ish
     orbit_data = {}  # keys are datetime as str, values arrays of XY
  
     for orbit in orbits:
         s0, s1 = _sta_sto(orbit, started_at, stopped_at)
  
+        nc_keys = default_nc_keys.copy()
+        if 'parameters' in orbit:
+            nc_keys.update(orbit['parameters'])
+
         orbit_files = retrieve_amda_netcdf(
             target_config['slug'], orbit['slug'], s0, s1
         )
@@ -457,19 +482,16 @@ def get_data_for_target(target_config, started_at, stopped_at):
                       (target_config['name'], orbit_file))
             cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
             times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
-            try:
-                data_hee = cdf_handle.variables['HEE']
-            except KeyError:
-                data_hee = cdf_handle.variables['XYZ_HEE']  # p67 uses this
+            data_hee = _read_var(cdf_handle, nc_keys, 'hee', mandatory=True)
  
             log.debug("%s: aggregating data from '%s'..." %
                       (target_config['name'], orbit_file))
-            for time, datum_hee in zip(times, data_hee):
+            for ltime, datum_hee in zip(times, data_hee):
                 try:
-                    dtime = datetime_from_list(time)
-                except Exception as e:
-                    log.error("Failed to parse time from %s." % time)
-                    raise e
+                    dtime = datetime_from_list(ltime)
+                except Exception:
+                    log.error("Failed to parse time from %s." % ltime)
+                    raise
                 if s0 <= dtime <= s1:
                     dkey = round_time(dtime, 60*60).strftime(precision)
                     orbit_data[dkey] = datum_hee
@@ -481,40 +503,66 @@ def get_data_for_target(target_config, started_at, stopped_at):
         model_files = retrieve_amda_netcdf(
             target_config['slug'], model['slug'], s0, s1
         )
+        nc_keys = default_nc_keys.copy()
+        if 'parameters' in model:
+            nc_keys.update(model['parameters'])
+
         for model_file in model_files:
-            # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
             log.debug("%s: opening model NETCDF4 '%s'..." %
                       (target_config['name'], model_file))
             cdf_handle = Dataset(model_file, "r", format="NETCDF4")
+            # log.debug(cdf_handle.variables.keys())
             times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
-            data_v = cdf_handle.variables['V']
-            data_b = cdf_handle.variables['B']
-            data_t = cdf_handle.variables['T']
-            data_n = cdf_handle.variables['N']
-            data_p = cdf_handle.variables['P_dyn']
-            data_d = cdf_handle.variables['Delta_angle']
+            data_v = _read_var(cdf_handle, nc_keys, 'vtot')
+            data_b = _read_var(cdf_handle, nc_keys, 'magn')
+            data_t = _read_var(cdf_handle, nc_keys, 'temp')
+            data_n = _read_var(cdf_handle, nc_keys, 'dens')
+            data_p = _read_var(cdf_handle, nc_keys, 'pdyn')
+            data_a = _read_var(cdf_handle, nc_keys, 'atse')
+
+            # Usually:
+            # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
+            # Earth:
+            # Time, BartelsNumber, ImfID, SwID, ImfPoints,
+            # SwPoints, B_M_av, B_Vec_av, B_Theta_av,
+            # B_Phi_av, B, T, N, V, Vlat, Vlon,
+            # Alpha, RamP, E, Beta, Ma, Kp, R, DST,
+            # AE, Flux, Flag, F10_Index, StartTime, StopTime
+
             log.debug("%s: aggregating data from '%s'..." %
                       (target_config['name'], model_file))
-            for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \
-                    in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
-                vrad = datum_v[0]
-                vtan = datum_v[1]
+            for ltime, datum_v, datum_b, datum_t, datum_n, datum_p, datum_a \
+                    in zip(times, data_v, data_b, data_t, data_n, data_p, data_a):
+
                 try:
-                    dtime = datetime_from_list(time)
-                except Exception as e:
-                    log.error("Failed to parse time from %s." % time)
-                    raise e
+                    dtime = datetime_from_list(ltime)
+                except Exception:
+                    log.error("Failed to parse time from %s." % ltime)
+                    raise
                 if s0 <= dtime <= s1:
                     dkey = round_time(dtime, 60*60).strftime(precision)
+
+                    if hasattr(datum_v, '__len__'):
+                        vrad = datum_v[0]
+                        vtan = datum_v[1]
+                        vtot = sqrt(vrad * vrad + vtan * vtan)
+                    else:  # eg: Earth
+                        vrad = None
+                        vtan = None
+                        vtot = datum_v
+
                     x_hee = None
                     y_hee = None
+                    if target_config['slug'] == 'earth':
+                        x_hee = 1
+                        y_hee = 0
                     if dkey in orbit_data:
                         x_hee = orbit_data[dkey][0]
                         y_hee = orbit_data[dkey][1]
                     all_data[dkey] = (
                         dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-                        vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
-                        datum_b, datum_t, datum_n, datum_p, datum_d,
+                        vrad, vtan, vtot,
+                        datum_b, datum_t, datum_n, datum_p, datum_a,
                         x_hee, y_hee
                     )
             cdf_handle.close()
@@ -565,7 +613,12 @@ def generate_csv_file_if_needed(target_slug, started_at, stopped_at):
                                               stopped_at=stopped_at))
             log.info("Generation of '%s' done." % filename)
         except Exception as e:
+            from sys import exc_info
+            from traceback import extract_tb
+            exc_type, exc_value, exc_traceback = exc_info()
             log.error(e)
+            for trace in extract_tb(exc_traceback):
+                log.error(trace)
             if isfile(local_csv_file):
                 log.warn("Removing failed CSV '%s'..." % local_csv_file)
                 removefile(local_csv_file)
@@ -875,9 +928,9 @@ def download_targets_netcdf(targets, params, started_at, stopped_at):
                 nc_y[:] = values_y
             log.debug("Writing NetCDF '%s'..." % nc_filename)
  
-        except Exception as e:
+        except Exception:
             log.error("Failed to generate NetCDF '%s'." % nc_filename)
-            raise e
+            raise
         finally:
             nc_handle.close()
  
@@ -199,12 +199,15 @@
             var dtime;
             dtime = timeFormat(d['time']);
             configuration['parameters'].forEach(function(parameter){
-              var id;
+              var id, val;
               id = parameter['id'];
-              return data[id].push({
-                x: dtime,
-                y: parseFloat(d[id])
-              });
+              val = parseFloat(d[id]);
+              if (!isNaN(val)) {
+                return data[id].push({
+                  x: dtime,
+                  y: val
+                });
+              }
             });
             if (d['xhee'] && d['yhee']) {
               return data['hee'].push({
@@ -299,10 +302,13 @@
         if (!(id in data)) {
           console.error("No data for id '" + id + "'.", data);
         }
-        return this$.time_series.push(new TimeSeries(id, title, target, data[id], this$.parameters[id].active, container, {
-          'started_at': this$.started_at,
-          'stopped_at': this$.stopped_at
-        }));
+        console.log(target['name'], id, data[id]);
+        if (data[id].length) {
+          return this$.time_series.push(new TimeSeries(id, title, target, data[id], this$.parameters[id].active, container, {
+            'started_at': this$.started_at,
+            'stopped_at': this$.stopped_at
+          }));
+        }
       });
       this.time_series.forEach(function(ts){
         ts.options['onMouseOver'] = function(){
@@ -169,7 +169,9 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
           dtime = timeFormat(d['time'])
           configuration['parameters'].forEach((parameter) ->
             id = parameter['id']
-            data[id].push({x: dtime, y: parseFloat(d[id])})
+            val = parseFloat(d[id])
+            if not isNaN(val)
+              data[id].push({x: dtime, y: val})
           )
           if d['xhee'] and d['yhee']
             data['hee'].push({
@@ -250,12 +252,14 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
       container = @configuration['time_series_container']
       id = parameter['id'] ; title = parameter['title']
       if id not of data then console.error("No data for id '#{id}'.", data)
-      @time_series.push(new TimeSeries(
-        id, title, target, data[id], @parameters[id].active, container, {
-          'started_at': @started_at,
-          'stopped_at': @stopped_at,
-        }
-      ))
+      console.log(target['name'], id, data[id])
+      if data[id].length
+        @time_series.push(new TimeSeries(
+          id, title, target, data[id], @parameters[id].active, container, {
+            'started_at': @started_at,
+            'stopped_at': @stopped_at,
+          }
+        ))
     )
     @time_series.forEach((ts) ~>  # returning true may be faster
       ts.options['onMouseOver'] = ~>
@@ -11,7 +11,7 @@
     <div class="target {{ target.slug }} {{ 'locked' if target.locked else 'active' }}"
          data-target-slug="{{ target.slug }}">
       <img width="64px" height="64px"
-           src="{{ static('img/target/'~target.slug~'_128.png') }}"
+           src="{{ static('img/target/'~target.slug~'_256.png') }}"
            title="{{ target.title }}"
            alt="{{ target.name }}">
       <img width="64px" height="64px"