From 297a7dfcb69e43cf00a86b585bb78fea2032ab6a Mon Sep 17 00:00:00 2001 From: Goutte Date: Mon, 27 Nov 2017 11:46:33 +0100 Subject: [PATCH] Add support for inputs STEREO-A, STEREO-B, L1 (needs testing) --- CHANGELOG.md | 4 ++-- config.yml | 182 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------------------- web/run.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------- web/view/home.html.jinja2 | 32 +++++++++++++++++++++++++++++--- 4 files changed, 240 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f45c84..a07e04b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,12 @@ ## Nicolas - [ ] séries temporelles en échelle log ou linéaire +- [ ] quant on zoome modifier l'échelle en Y afin qu'elle s'adapte aux valeurs maximales de l'intervalle du zoom - [ ] permettre à l'utilisateur de choisir inner heliosphere (jusqu'à mars) ou outer heliosphere (jusqu'à saturne) pour la visualisation des éphémérides planétaires - [ ] rajouter uranus et neptune comme planètes cibles - [ ] rajouter MAVEN, New Horizons, Cassini, Galileo, ExoMars comme spacecraft cible - [ ] prévoir la possibilité à l'utilisateur d'afficher le temps en Day Of Year -- [ ] quant on zoome modifier l'échelle en Y afin qu'elle s'adapte aux valeurs maximales de l'intervalle du zoom - [ ] modifier le bandeau vertical d'affichage des choix utilisateurs pour qu'il soit plus petit (en particulier les champs paramètres) -- [ ] ajouter dans le bandeau vertical un onglet INPUT DATA avec 3 choix possibles: STEREO-A, STEREO-B, L1 - [ ] ajouter dans le bandeau vertical un onglet LAYERS avec 3 choix possibles: Jupiter Thermosphere Model, HST Jupiter Observations, HST Saturn Observations - [ ] ajouter un bouton ALERT qui permet d'ouvrir une nouvelle page de formulaire ou l'utilisateur entrerait les données suivantes: * email @@ -32,6 +31,7 @@ - [ ] Rework the images of Rosetta and Juno - [ ] Enable p67 +- [x] Add support for inputs STEREO-A, STEREO-B, L1 - [x] More ticks along X-axis (I tried, but it does not behave as expected) - [x] Smaller footer credits - [x] Move the version to the footer diff --git a/config.yml b/config.yml index 97ac075..cb24ae4 100644 --- a/config.yml +++ b/config.yml @@ -46,6 +46,17 @@ authors: amda: "http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet={dataSet}&StartTime={startTime}&StopTime={stopTime}" +inputs: + - slug: "l1" + name: "L1" + desc: "The default input." + - slug: "sa" + name: "STEREO-A" + desc: "The first of the STEREO (Solar Terrestrial Relations Observatory), a solar observation mission." + - slug: "sb" + name: "STEREO-B" + desc: "The second of the STEREO (Solar Terrestrial Relations Observatory), a solar observation mission." + #Targets #Earth earth_orb_all #Jupiter jupiter_orb_all @@ -75,12 +86,17 @@ targets: name: 'Mercury' title: 'Mercury' orbit: - models: - - slug: 'mercury_orb_all' - semimajor: 0 - semiminor: 0 + models: + - slug: 'mercury_orb_all' + semimajor: 0 + semiminor: 0 models: - - slug: 'tao_mercury_sw' + l1: + - slug: 'tao_mercury_sw' + sa: + - slug: 'tao_mercury_sw' + sb: + - slug: 'tao_mercury_sw' locked: false default: true - type: 'planet' @@ -88,12 +104,17 @@ targets: name: 'Venus' title: 'Venus' orbit: - models: - - slug: 'venus_orb_all' - semimajor: 0.72333199 - semiminor: 0.7233154 + models: + - slug: 'venus_orb_all' + semimajor: 0.72333199 + semiminor: 0.7233154 models: - - slug: 'tao_venus_sw' + l1: + - slug: 'tao_venus_sw' + sa: + - slug: 'tao_venus_sw' + sb: + - slug: 'tao_venus_sw' locked: false default: true - type: 'planet' @@ -101,42 +122,71 @@ targets: name: 'Earth' title: 'Earth' orbit: - models: [] # Earth has no orbit models, we hard-coded it to (1, 0) + models: [] # Earth has no orbit models, we hard-coded it to (1, 0) models: - - slug: 'omni_hour_all' - - slug: 'ace_swepam_real' - parameters: - dens: 'Dens' - vtot: 'Vel' - temp: 'Temp' + l1: + - slug: 'omni_hour_all' + - slug: 'ace_swepam_real' + parameters: + dens: 'Dens' + vtot: 'Vel' + temp: 'Temp' + sa: + - slug: 'omni_hour_all' + - slug: 'ace_swepam_real' + parameters: + dens: 'Dens' + vtot: 'Vel' + temp: 'Temp' + sb: + - slug: 'omni_hour_all' + - slug: 'ace_swepam_real' + parameters: + dens: 'Dens' + vtot: 'Vel' + temp: 'Temp' locked: false default: true - type: 'planet' slug: 'mars' name: 'Mars' title: 'Mars' - models: - - slug: 'tao_mars_sw' - - slug: 'tao_mars_swrt' orbit: - models: - - slug: 'mars_orb_all' - semimajor: 1.52366231 - semiminor: 1.51700011 + models: + - slug: 'mars_orb_all' + semimajor: 1.52366231 + semiminor: 1.51700011 + models: + l1: + - slug: 'tao_mars_sw' + - slug: 'tao_mars_swrt' + sa: + - slug: 'tao_mars_sw' + - slug: 'tao_mars_swrt' + sb: + - slug: 'tao_mars_sw' + - slug: 'tao_mars_swrt' locked: false default: true - type: 'planet' slug: 'jupiter' name: 'Jupiter' title: 'Jupiter' - models: - - slug: 'tao_jup_sw' - - slug: 'tao_jup_swrt' orbit: - models: - - slug: 'jupiter_orb_all' - semimajor: 5.45516759 - semiminor: 4.95155843 + models: + - slug: 'jupiter_orb_all' + semimajor: 5.45516759 + semiminor: 4.95155843 + models: + l1: + - slug: 'tao_jup_sw' + - slug: 'tao_jup_swrt' + sa: + - slug: 'tao_jup_sw' + - slug: 'tao_jup_swrt' + sb: + - slug: 'tao_jup_sw' + - slug: 'tao_jup_swrt' locked: false default: true - type: 'planet' @@ -144,12 +194,17 @@ targets: name: 'Saturn' title: 'Saturn' orbit: - models: - - slug: 'saturn_orb_all' - semimajor: 9.53707032 - semiminor: 9.5230773 + models: + - slug: 'saturn_orb_all' + semimajor: 9.53707032 + semiminor: 9.5230773 models: - - slug: 'tao_sat_sw' + l1: + - slug: 'tao_sat_sw' + sa: + - slug: 'tao_sat_sw' + sb: + - slug: 'tao_sat_sw' locked: false default: true - type: 'probe' @@ -157,16 +212,21 @@ targets: name: 'Rosetta' title: 'Rosetta' orbit: - models: - - slug: 'ros_orb_cruise' - stopped_at: '2014-08-02T00:00:00' - - slug: 'p67_orb_all' - started_at: '2014-08-02T00:00:00' - #stopped_at: '2016-09-30T11:19:00' - parameters: - hee: 'XYZ_HEE' + models: + - slug: 'ros_orb_cruise' + stopped_at: '2014-08-02T00:00:00' + - slug: 'p67_orb_all' + started_at: '2014-08-02T00:00:00' + #stopped_at: '2016-09-30T11:19:00' + parameters: + hee: 'XYZ_HEE' models: - - slug: 'tao_ros_sw' + l1: + - slug: 'tao_ros_sw' + sa: + - slug: 'tao_ros_sw' + sb: + - slug: 'tao_ros_sw' locked: false default: false - type: 'probe' @@ -174,13 +234,18 @@ targets: name: 'Juno' title: 'Juno' orbit: - models: - - slug: 'juno_cruise_all' - stopped_at: '2016-07-05T03:53:00' - - slug: 'jupiter_orb_all' - started_at: '2016-07-05T03:53:00' + models: + - slug: 'juno_cruise_all' + stopped_at: '2016-07-05T03:53:00' + - slug: 'jupiter_orb_all' + started_at: '2016-07-05T03:53:00' models: - - slug: 'tao_juno_sw' + l1: + - slug: 'tao_juno_sw' + sa: + - slug: 'tao_juno_sw' + sb: + - slug: 'tao_juno_sw' locked: false default: false - type: 'comet' @@ -188,12 +253,17 @@ targets: name: 'Churyumov-Gerasimenko' title: 'Churyumov-Gerasimenko (coming soon)' orbit: - models: - - slug: 'p67_orb_all' - parameters: - hee: 'XYZ_HEE' + models: + - slug: 'p67_orb_all' + parameters: + hee: 'XYZ_HEE' models: - - slug: 'tao_p67_sw' # fixme + l1: + - slug: 'tao_p67_sw' # fixme + sa: + - slug: 'tao_p67_sw' # fixme + sb: + - slug: 'tao_p67_sw' # fixme locked: true default: false diff --git a/web/run.py b/web/run.py index abe194e..d43afda 100755 --- a/web/run.py +++ b/web/run.py @@ -419,14 +419,17 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): return list(set(local_netc_files)) # remove possible dupes -def get_data_for_target(target_config, started_at, stopped_at): +def get_data_for_target( + target_config, + started_at, stopped_at, + input_slug='l1'): """ :return: dict whose keys are datetime as str, values tuples of data """ log.debug("Grabbing data for '%s'..." % target_config['slug']) try: - models = target_config['models'] + models = target_config['models'][input_slug] except Exception as e: abort(500, "Invalid model configuration for '%s' : %s" % (target_config['slug'], str(e))) @@ -459,7 +462,7 @@ def get_data_for_target(target_config, started_at, stopped_at): raise Exception("No variable '%s' found in NetCDF." % _keys[_key]) return [None] * len(_nc.variables['Time']) # slow -- use numpy! - # Override them using the configuration, maybe also put these in config ? + # Override these using the model configuration default_nc_keys = { 'hee': 'HEE', 'vtot': 'V', @@ -576,14 +579,17 @@ def get_data_for_target(target_config, started_at, stopped_at): return all_data -def generate_csv_contents(target_slug, started_at, stopped_at): +def generate_csv_contents(target_slug, input_slug, started_at, stopped_at): target_config = get_target_config(target_slug) log.debug("Crunching CSV contents for '%s'..." % target_config['name']) si = StringIO.StringIO() cw = csv_writer(si) cw.writerow(PROPERTIES) - all_data = get_data_for_target(target_config, started_at, stopped_at) + all_data = get_data_for_target( + target_config=target_config, input_slug=input_slug, + started_at=started_at, stopped_at=stopped_at + ) log.debug("Writing and sorting CSV for '%s'..." % target_config['slug']) for dkey in sorted(all_data): @@ -593,15 +599,15 @@ def generate_csv_contents(target_slug, started_at, stopped_at): return si.getvalue() -def generate_csv_file_if_needed(target_slug, started_at, stopped_at): - filename = "%s_%s_%s.csv" % (target_slug, - started_at.strftime(FILE_DATE_FMT), - stopped_at.strftime(FILE_DATE_FMT)) +def generate_csv_file_if_needed(target_slug, input_slug, started_at, stopped_at): + filename = "%s_%s_%s_%s.csv" % (target_slug, input_slug, + started_at.strftime(FILE_DATE_FMT), + stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = join(CACHE_DIR, filename) generate = True if isfile(local_csv_file): - # It need to have more than one line to not be empty (headers) + # It needs to have more than one line to not be empty (headers) with open(local_csv_file) as f: cnt = 0 for _ in f: @@ -683,6 +689,16 @@ def remove_files_created_before(date, in_directory): return removed_files +def get_input_slug_from_query(inp=None): + if inp is None: + input_slug = request.args.get('input_slug', 'l1') + else: + input_slug = inp + if input_slug not in [i.slug for i in config.inputs]: + input_slug = 'l1' # be tolerant instead of yelling loudly + return input_slug + + def get_hit_counter(): hit_count_path = get_path("../VISITS") @@ -730,9 +746,11 @@ def home(): increment_hit_counter() parameters = PARAMETERS.values() parameters.sort(key=lambda x: x['position']) + input_slug = get_input_slug_from_query() return render_view('home.html.jinja2', { 'targets': config['targets'], 'parameters': parameters, + 'input_slug': input_slug, 'planets': [s for s in config['targets'] if s['type'] == 'planet'], 'probes': [s for s in config['targets'] if s['type'] == 'probe'], 'comets': [s for s in config['targets'] if s['type'] == 'comet'], @@ -740,12 +758,13 @@ def home(): }) -@app.route("/__.csv") -def download_target_csv(target, started_at, stopped_at): +@app.route("/___.csv") +def download_target_csv(target, inp, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a CSV file. `started_at` and `stopped_at` should be UTC. + `inp` is the input slug, l1 or sa or sb. """ check_target_config(target) try: @@ -756,20 +775,24 @@ def download_target_csv(target, started_at, stopped_at): stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT) except: abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) + input_slug = get_input_slug_from_query(inp=inp) - filename = "%s_%s_%s.csv" % (target, - started_at.strftime(FILE_DATE_FMT), - stopped_at.strftime(FILE_DATE_FMT)) + filename = "%s_%s_%s_%s.csv" % (target, input_slug, + started_at.strftime(FILE_DATE_FMT), + stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = join(CACHE_DIR, filename) - generate_csv_file_if_needed(target, started_at, stopped_at) + generate_csv_file_if_needed( + target_slug=target, input_slug=input_slug, + started_at=started_at, stopped_at=stopped_at + ) if not isfile(local_csv_file): abort(500, "Could not cache CSV file at '%s'." % local_csv_file) return send_from_directory(CACHE_DIR, filename) -@app.route("/__.tar.gz") -def download_targets_tarball(targets, started_at, stopped_at): +@app.route("/___.tar.gz") +def download_targets_tarball(targets, inp, started_at, stopped_at): """ Grab data and orbit data for each of the specified `targets`, in their own CSV file, and make a tarball of them. @@ -801,24 +824,35 @@ def download_targets_tarball(targets, started_at, stopped_at): sta = started_at.strftime(FILE_DATE_FMT) sto = stopped_at.strftime(FILE_DATE_FMT) - gzip_filename = "%s_%s_%s.tar.gz" % (separator.join(targets), sta, sto) + input_slug = get_input_slug_from_query(inp=inp) + + gzip_filename = "%s_%s_%s_%s.tar.gz" % ( + separator.join(targets), input_slug, sta, sto + ) local_gzip_file = join(CACHE_DIR, gzip_filename) if not isfile(local_gzip_file): log.debug("Creating the CSV files for the tarball...") for target_config in targets_configs: - filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto) + filename = "%s_%s_%s_%s.csv" % ( + target_config['slug'], input_slug, sta, sto + ) local_csv_file = join(CACHE_DIR, filename) if not isfile(local_csv_file): with open(local_csv_file, mode="w+") as f: - f.write(generate_csv_contents(target_config['slug'], - started_at=started_at, - stopped_at=stopped_at)) + f.write(generate_csv_contents( + target_slug=target_config['slug'], + started_at=started_at, + stopped_at=stopped_at, + input_slug=input_slug + )) log.debug("Creating the tarball '%s'..." % local_gzip_file) with tarfile.open(local_gzip_file, "w:gz") as tar: for target_config in targets_configs: - filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto) + filename = "%s_%s_%s_%s.csv" % ( + target_config['slug'], input_slug, sta, sto + ) local_csv_file = join(CACHE_DIR, filename) tar.add(local_csv_file, arcname=filename) @@ -828,8 +862,8 @@ def download_targets_tarball(targets, started_at, stopped_at): return send_from_directory(CACHE_DIR, gzip_filename) -@app.route("/___.nc") -def download_targets_netcdf(targets, params, started_at, stopped_at): +@app.route("/____.nc") +def download_targets_netcdf(targets, inp, params, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a NetCDF file. @@ -867,8 +901,11 @@ def download_targets_netcdf(targets, params, started_at, stopped_at): sta = started_at.strftime(date_fmt) sto = stopped_at.strftime(date_fmt) - nc_filename = "%s_%s_%s_%s.nc" % \ - (separator.join(targets), separator.join(params), sta, sto) + input_slug = get_input_slug_from_query(inp=inp) + + nc_filename = "%s_%s_%s_%s_%s.nc" % ( + separator.join(targets), separator.join(params), input_slug, sta, sto + ) nc_path = join(CACHE_DIR, nc_filename) if not isfile(nc_path): @@ -883,7 +920,10 @@ def download_targets_netcdf(targets, params, started_at, stopped_at): target_slug = target['slug'] log.debug("Adding group '%s' to the NetCDF..." % target_slug) nc_group = nc_handle.createGroup(target_slug) - data = get_data_for_target(target, started_at, stopped_at) + data = get_data_for_target( + target_config=target, input_slug=input_slug, + started_at=started_at, stopped_at=stopped_at + ) dkeys = sorted(data) dimension = 'dim_'+target_slug nc_handle.createDimension(dimension, len(dkeys)) @@ -946,8 +986,8 @@ def download_targets_netcdf(targets, params, started_at, stopped_at): return send_from_directory(CACHE_DIR, nc_filename) -@app.route("/__.cdf") -def download_targets_cdf(targets, started_at, stopped_at): +@app.route("/___.cdf") +def download_targets_cdf(targets, inp, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a CDF file. @@ -986,7 +1026,11 @@ def download_targets_cdf(targets, started_at, stopped_at): sta = started_at.strftime(FILE_DATE_FMT) sto = stopped_at.strftime(FILE_DATE_FMT) - cdf_filename = "%s_%s_%s.cdf" % (separator.join(targets), sta, sto) + input_slug = get_input_slug_from_query(inp=inp) + + cdf_filename = "%s_%s_%s_%s.cdf" % ( + separator.join(targets), input_slug, sta, sto + ) cdf_path = join(CACHE_DIR, cdf_filename) if not isfile(cdf_path): @@ -1010,7 +1054,10 @@ def download_targets_cdf(targets, started_at, stopped_at): available_params = list(PROPERTIES) for target in targets_configs: target_slug = target['slug'] - data = get_data_for_target(target, started_at, stopped_at) + data = get_data_for_target( + target_config=target, input_slug=input_slug, + started_at=started_at, stopped_at=stopped_at + ) dkeys = sorted(data) values = [] @@ -1108,13 +1155,14 @@ def cache_warmup(): stopped_at = today + after sta = started_at.strftime(FILE_DATE_FMT) sto = stopped_at.strftime(FILE_DATE_FMT) + inp = 'l1' # default input, maybe warm them all up ? targets = get_active_targets() for target in targets: - download_target_csv(target['slug'], sta, sto) + download_target_csv(target['slug'], inp, sta, sto) targets_slugs = [target['slug'] for target in targets] targets_slugs.sort() - download_targets_cdf('-'.join(targets_slugs), sta, sto) + download_targets_cdf('-'.join(targets_slugs), inp, sta, sto) return "Done" diff --git a/web/view/home.html.jinja2 b/web/view/home.html.jinja2 index 65df6ba..880c248 100755 --- a/web/view/home.html.jinja2 +++ b/web/view/home.html.jinja2 @@ -59,7 +59,9 @@
+ {{ icon('calendar') }} Time Interval +
@@ -96,6 +98,22 @@
+{# {{ icon('database') }} Inputs#} +{##} +{#
#} +{#{% for input in config.inputs %}#} +{# #} +{#
#} +{#{% endfor %}#} +{#
#} +{# #} +{#
#} +{##} +{#
#} + {{ icon('flask') }} Parameters