From d1c44c51ee5a6d366acecb225cb0d8752aeeace8 Mon Sep 17 00:00:00 2001 From: Goutte Date: Wed, 4 Oct 2017 17:42:20 +0200 Subject: [PATCH] Enable Earth Enable more precise configuration of each models' parameters --- CHANGELOG.md | 11 ++++++++++- README.md | 1 + config.yml | 20 +++++++++++++------- web/run.py | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------- web/static/js/swapp.js | 24 +++++++++++++++--------- web/static/js/swapp.ls | 18 +++++++++++------- web/view/home.html.jinja2 | 2 +- 7 files changed, 138 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adff0f..15d6cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,22 @@ - [ ] Add a README to the download tarball - [ ] Set the log level to _error_ in production (see `web/run.py`) - [ ] CRON statements to call the cache cleanup and warmup -- [ ] Cache warmup (generate for today's default interval) `/cache/warmup` - [ ] Give the future data another color An heliospheric propagation 1D MHD model for solar wind prediction at planets, probes and comets. +## 1.0.0-rc5 + +- [ ] Rework the images of Rosetta and Juno +- [ ] Cache warmup (generate for today's default interval) `/cache/warmup` +- [ ] Improve the generated CDF for SAMP +- [ ] Enable p67 +- [x] Enable Earth +- [x] Enable more precise configuration of each models' parameters + + ## 1.0.0-rc4 - [x] Sort times series by closeness to the sun diff --git a/README.md b/README.md index a4c8694..476ceec 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ It also gathers NetCDF data from AMDA, and serves it as CSV to the plotter. - `config.yml` : the main configuration file. - `web/run.py` : the front controller, holding most of the code. - `web/view/home.html.jinja2` : the HTML template. +- `web/static/js/swapp.ls` : most of the javascript client-side. ## Install diff --git a/config.yml b/config.yml index 9c6fd00..15be444 100644 --- a/config.yml +++ b/config.yml @@ -98,15 +98,17 @@ targets: - type: 'planet' slug: 'earth' name: 'Earth' - title: 'Earth (coming soon)' + title: 'Earth' orbit: - models: - - slug: 'earth_orb_all' - semimajor: 0 - semiminor: 0 + models: [] # Earth has no orbit models, we hard-coded it to (1, 0) models: - - slug: 'tao_earth_sw' - locked: true + - slug: 'omni_hour_all' + - slug: 'ace_swepam_real' + parameters: + dens: 'Dens' + vtot: 'Vel' + temp: 'Temp' + locked: false default: true - type: 'planet' slug: 'mars' @@ -159,6 +161,8 @@ targets: stopped_at: '2014-08-02T00:00:00' - slug: 'p67_orb_all' started_at: '2014-08-02T00:00:00' + parameters: + hee: 'XYZ_HEE' models: - slug: 'tao_ros_sw' locked: false @@ -184,6 +188,8 @@ targets: orbit: models: - slug: 'p67_orb_all' + parameters: + hee: 'XYZ_HEE' locked: true default: false diff --git a/web/run.py b/web/run.py index 4114908..dde3e84 100755 --- a/web/run.py +++ b/web/run.py @@ -74,7 +74,7 @@ CACHE_DIR = get_path('../cache') # innermost loop of `get_data_for_target`. # The javascript knows the targets' properties under these names. PROPERTIES = ('time', 'vrad', 'vtan', 'vtot', 'btan', 'temp', 'pdyn', 'dens', - 'angl', 'xhee', 'yhee') + 'atse', 'xhee', 'yhee') # The parameters that the users can handle. # The slug MUST be one of the properties above. @@ -119,8 +119,8 @@ PARAMETERS = { 'active': False, 'position': 50, }, - 'angl': { - 'slug': 'angl', + 'atse': { + 'slug': 'atse', 'name': 'Angle T-S-E', 'title': 'Angle Target-Sun-Earth.', 'units': 'deg', @@ -426,9 +426,10 @@ def get_data_for_target(target_config, started_at, stopped_at): % (target_config['slug'], str(e))) try: orbits = target_config['orbit']['models'] - except Exception as e: - abort(500, "Invalid orbit configuration for '%s' : %s" - % (target_config['slug'], str(e))) + except KeyError as e: + orbits = [] + # abort(500, "Invalid orbit configuration for '%s' : %s" + # % (target_config['slug'], str(e))) def _sta_sto(_cnf, _sta, _sto): if 'started_at' in _cnf: @@ -443,12 +444,36 @@ def get_data_for_target(target_config, started_at, stopped_at): _s1 = _sto return _s0, _s1 + def _read_var(_nc, _keys, _key, mandatory=False): + try: + return _nc.variables[_keys[_key]] + except KeyError: + pass + if mandatory: + raise Exception("No variable '%s' found in NetCDF." % _keys[_key]) + return [None] * len(_nc.variables['Time']) # slow -- use numpy! + + # Override them using the configuration, maybe also put these in config ? + default_nc_keys = { + 'hee': 'HEE', + 'vtot': 'V', + 'magn': 'B', + 'temp': 'T', + 'dens': 'N', + 'pdyn': 'P_dyn', + 'atse': 'Delta_angle', + } + precision = "%Y-%m-%dT%H" # model and orbits times are only equal-ish orbit_data = {} # keys are datetime as str, values arrays of XY for orbit in orbits: s0, s1 = _sta_sto(orbit, started_at, stopped_at) + nc_keys = default_nc_keys.copy() + if 'parameters' in orbit: + nc_keys.update(orbit['parameters']) + orbit_files = retrieve_amda_netcdf( target_config['slug'], orbit['slug'], s0, s1 ) @@ -457,19 +482,16 @@ def get_data_for_target(target_config, started_at, stopped_at): (target_config['name'], orbit_file)) cdf_handle = Dataset(orbit_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms - try: - data_hee = cdf_handle.variables['HEE'] - except KeyError: - data_hee = cdf_handle.variables['XYZ_HEE'] # p67 uses this + data_hee = _read_var(cdf_handle, nc_keys, 'hee', mandatory=True) log.debug("%s: aggregating data from '%s'..." % (target_config['name'], orbit_file)) - for time, datum_hee in zip(times, data_hee): + for ltime, datum_hee in zip(times, data_hee): try: - dtime = datetime_from_list(time) - except Exception as e: - log.error("Failed to parse time from %s." % time) - raise e + dtime = datetime_from_list(ltime) + except Exception: + log.error("Failed to parse time from %s." % ltime) + raise if s0 <= dtime <= s1: dkey = round_time(dtime, 60*60).strftime(precision) orbit_data[dkey] = datum_hee @@ -481,40 +503,66 @@ def get_data_for_target(target_config, started_at, stopped_at): model_files = retrieve_amda_netcdf( target_config['slug'], model['slug'], s0, s1 ) + nc_keys = default_nc_keys.copy() + if 'parameters' in model: + nc_keys.update(model['parameters']) + for model_file in model_files: - # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn log.debug("%s: opening model NETCDF4 '%s'..." % (target_config['name'], model_file)) cdf_handle = Dataset(model_file, "r", format="NETCDF4") + # log.debug(cdf_handle.variables.keys()) times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms - data_v = cdf_handle.variables['V'] - data_b = cdf_handle.variables['B'] - data_t = cdf_handle.variables['T'] - data_n = cdf_handle.variables['N'] - data_p = cdf_handle.variables['P_dyn'] - data_d = cdf_handle.variables['Delta_angle'] + data_v = _read_var(cdf_handle, nc_keys, 'vtot') + data_b = _read_var(cdf_handle, nc_keys, 'magn') + data_t = _read_var(cdf_handle, nc_keys, 'temp') + data_n = _read_var(cdf_handle, nc_keys, 'dens') + data_p = _read_var(cdf_handle, nc_keys, 'pdyn') + data_a = _read_var(cdf_handle, nc_keys, 'atse') + + # Usually: + # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn + # Earth: + # Time, BartelsNumber, ImfID, SwID, ImfPoints, + # SwPoints, B_M_av, B_Vec_av, B_Theta_av, + # B_Phi_av, B, T, N, V, Vlat, Vlon, + # Alpha, RamP, E, Beta, Ma, Kp, R, DST, + # AE, Flux, Flag, F10_Index, StartTime, StopTime + log.debug("%s: aggregating data from '%s'..." % (target_config['name'], model_file)) - for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ - in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): - vrad = datum_v[0] - vtan = datum_v[1] + for ltime, datum_v, datum_b, datum_t, datum_n, datum_p, datum_a \ + in zip(times, data_v, data_b, data_t, data_n, data_p, data_a): + try: - dtime = datetime_from_list(time) - except Exception as e: - log.error("Failed to parse time from %s." % time) - raise e + dtime = datetime_from_list(ltime) + except Exception: + log.error("Failed to parse time from %s." % ltime) + raise if s0 <= dtime <= s1: dkey = round_time(dtime, 60*60).strftime(precision) + + if hasattr(datum_v, '__len__'): + vrad = datum_v[0] + vtan = datum_v[1] + vtot = sqrt(vrad * vrad + vtan * vtan) + else: # eg: Earth + vrad = None + vtan = None + vtot = datum_v + x_hee = None y_hee = None + if target_config['slug'] == 'earth': + x_hee = 1 + y_hee = 0 if dkey in orbit_data: x_hee = orbit_data[dkey][0] y_hee = orbit_data[dkey][1] all_data[dkey] = ( dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), - vrad, vtan, sqrt(vrad * vrad + vtan * vtan), - datum_b, datum_t, datum_n, datum_p, datum_d, + vrad, vtan, vtot, + datum_b, datum_t, datum_n, datum_p, datum_a, x_hee, y_hee ) cdf_handle.close() @@ -565,7 +613,12 @@ def generate_csv_file_if_needed(target_slug, started_at, stopped_at): stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) except Exception as e: + from sys import exc_info + from traceback import extract_tb + exc_type, exc_value, exc_traceback = exc_info() log.error(e) + for trace in extract_tb(exc_traceback): + log.error(trace) if isfile(local_csv_file): log.warn("Removing failed CSV '%s'..." % local_csv_file) removefile(local_csv_file) @@ -875,9 +928,9 @@ def download_targets_netcdf(targets, params, started_at, stopped_at): nc_y[:] = values_y log.debug("Writing NetCDF '%s'..." % nc_filename) - except Exception as e: + except Exception: log.error("Failed to generate NetCDF '%s'." % nc_filename) - raise e + raise finally: nc_handle.close() diff --git a/web/static/js/swapp.js b/web/static/js/swapp.js index d79da60..3134fde 100644 --- a/web/static/js/swapp.js +++ b/web/static/js/swapp.js @@ -199,12 +199,15 @@ var dtime; dtime = timeFormat(d['time']); configuration['parameters'].forEach(function(parameter){ - var id; + var id, val; id = parameter['id']; - return data[id].push({ - x: dtime, - y: parseFloat(d[id]) - }); + val = parseFloat(d[id]); + if (!isNaN(val)) { + return data[id].push({ + x: dtime, + y: val + }); + } }); if (d['xhee'] && d['yhee']) { return data['hee'].push({ @@ -299,10 +302,13 @@ if (!(id in data)) { console.error("No data for id '" + id + "'.", data); } - return this$.time_series.push(new TimeSeries(id, title, target, data[id], this$.parameters[id].active, container, { - 'started_at': this$.started_at, - 'stopped_at': this$.stopped_at - })); + console.log(target['name'], id, data[id]); + if (data[id].length) { + return this$.time_series.push(new TimeSeries(id, title, target, data[id], this$.parameters[id].active, container, { + 'started_at': this$.started_at, + 'stopped_at': this$.stopped_at + })); + } }); this.time_series.forEach(function(ts){ ts.options['onMouseOver'] = function(){ diff --git a/web/static/js/swapp.ls b/web/static/js/swapp.ls index 2bca06d..f96e79e 100644 --- a/web/static/js/swapp.ls +++ b/web/static/js/swapp.ls @@ -169,7 +169,9 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE dtime = timeFormat(d['time']) configuration['parameters'].forEach((parameter) -> id = parameter['id'] - data[id].push({x: dtime, y: parseFloat(d[id])}) + val = parseFloat(d[id]) + if not isNaN(val) + data[id].push({x: dtime, y: val}) ) if d['xhee'] and d['yhee'] data['hee'].push({ @@ -250,12 +252,14 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE container = @configuration['time_series_container'] id = parameter['id'] ; title = parameter['title'] if id not of data then console.error("No data for id '#{id}'.", data) - @time_series.push(new TimeSeries( - id, title, target, data[id], @parameters[id].active, container, { - 'started_at': @started_at, - 'stopped_at': @stopped_at, - } - )) + console.log(target['name'], id, data[id]) + if data[id].length + @time_series.push(new TimeSeries( + id, title, target, data[id], @parameters[id].active, container, { + 'started_at': @started_at, + 'stopped_at': @stopped_at, + } + )) ) @time_series.forEach((ts) ~> # returning true may be faster ts.options['onMouseOver'] = ~> diff --git a/web/view/home.html.jinja2 b/web/view/home.html.jinja2 index 7b36316..fe055ce 100755 --- a/web/view/home.html.jinja2 +++ b/web/view/home.html.jinja2 @@ -11,7 +11,7 @@
{{ target.name }}