From aa7247d6dad03769723529cfea0fe56213d8d1c2 Mon Sep 17 00:00:00 2001 From: Goutte Date: Wed, 27 Sep 2017 18:09:47 +0200 Subject: [PATCH] Generate a CDF file (not NetCDF) Bump to v1.0.0-rc4 --- CHANGELOG.md | 4 ++-- VERSION | 2 +- requirements.txt | 2 ++ web/run.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- web/view/home.html.jinja2 | 7 ++++--- 5 files changed, 124 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e64088a..3adff0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,6 @@ - [ ] CRON statements to call the cache cleanup and warmup - [ ] Cache warmup (generate for today's default interval) `/cache/warmup` - [ ] Give the future data another color -- [ ] Sort times series by closeness to the sun -- [ ] Generate a CDF file (not NetCDF) An heliospheric propagation 1D MHD model for solar wind prediction at planets, probes and comets. @@ -16,6 +14,8 @@ An heliospheric propagation 1D MHD model for solar wind prediction at planets, p ## 1.0.0-rc4 +- [x] Sort times series by closeness to the sun +- [x] Generate a CDF file (not NetCDF) - [x] Normalize time interval for time series - [x] Make the footer images clickable - [x] Highlight the visits counter diff --git a/VERSION b/VERSION index 0fab938..290c8c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0-rc3 \ No newline at end of file +1.0.0-rc4 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1c85df1..3569ecc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,5 @@ python-slugify==1.2.4 PyYAML==3.12 six==1.10.0 Werkzeug==0.12 +# spacepy==0.7 # requires a lot of system deps, like fortran +CDF==0.32 \ No newline at end of file diff --git a/web/run.py b/web/run.py index 93f0e0a..29ff61e 100755 --- a/web/run.py +++ b/web/run.py @@ -9,6 +9,7 @@ import random import tarfile import time import urllib +import cdf from csv import writer as csv_writer from math import sqrt from os import environ, remove as removefile @@ -112,7 +113,7 @@ PARAMETERS = { 'slug': 'dens', 'name': 'Density', 'title': 'The density N.', - 'units': u'cm⁻³', + 'units': 'cm^-3', 'active': False, 'position': 50, }, @@ -438,7 +439,7 @@ def get_data_for_target(target_config, started_at, stopped_at): _s1 = min(_s1, _sto) else: _s1 = _sto - return _s1, _s0 + return _s0, _s1 precision = "%Y-%m-%dT%H" # model and orbits times are only equal-ish orbit_data = {} # keys are datetime as str, values arrays of XY @@ -770,7 +771,7 @@ def download_targets_tarball(targets, started_at, stopped_at): def download_targets_netcdf(targets, params, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, - rearrange it and return it as a CSV file. + rearrange it and return it as a NetCDF file. `started_at` and `stopped_at` are expected to be UTC. targets: string list of targets' slugs, separated by `-`. @@ -884,6 +885,117 @@ def download_targets_netcdf(targets, params, started_at, stopped_at): return send_from_directory(CACHE_DIR, nc_filename) +@app.route("/__.cdf") +def download_targets_cdf(targets, started_at, stopped_at): + """ + Grab data and orbit data for the specified `target`, + rearrange it and return it as a CDF file. + `started_at` and `stopped_at` are expected to be UTC. + + targets: string list of targets' slugs, separated by `-`. + params: string list of targets' parameters, separated by `-`. + """ + separator = '-' # /!\ this char should never be in target's slugs + targets = targets.split(separator) + targets.sort() + targets_configs = [] + for target in targets: + if not target: + abort(400, "Invalid targets format : `%s`." % targets) + targets_configs.append(get_target_config(target)) + if 0 == len(targets_configs): + abort(400, "No valid targets specified. What are you doing?") + + params = PARAMETERS.keys() + # params = params.split(separator) + # params.sort() + # if 0 == len(params): + # abort(400, "No valid parameters specified. What are you doing?") + # if not is_list_in_list(params, PARAMETERS.keys()): + # abort(400, "Some parameters are not recognized in '%s'." % str(params)) + + try: + started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) + except: + abort(400, "Invalid started_at parameter : '%s'." % started_at) + try: + stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT) + except: + abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) + sta = started_at.strftime(FILE_DATE_FMT) + sto = stopped_at.strftime(FILE_DATE_FMT) + + cdf_filename = "%s_%s_%s.cdf" % (separator.join(targets), sta, sto) + cdf_path = join(CACHE_DIR, cdf_filename) + + if not isfile(cdf_path): + log.debug("Creating the CDF file '%s'..." % cdf_filename) + try: + cdf_handle = cdf.archive() + description = "Model and orbit data for %s." % \ + ', '.join([t['name'] for t in targets_configs]) + cdf_handle.attributes['Description'] = description + cdf_handle.attributes['Author'] = "Heliopropa.irap.omp.eu (CDPP)" + cdf_handle.attributes['Created'] = str(time.ctime(time.time())) + + available_params = list(PROPERTIES) + for target in targets_configs: + target_slug = target['slug'] + data = get_data_for_target(target, started_at, stopped_at) + dkeys = sorted(data) + + values = [] + units = "hours since 1970-01-01 00:00:00" + calendar = "standard" + for dkey in dkeys: + time_as_string = data[dkey][0][:-6] # remove +00:00 tail + date = datetime.datetime.strptime(time_as_string, FILE_DATE_FMT) + values.append(date2num(date, units=units, calendar=calendar)) + k = "%s_time" % target_slug + cdf_handle[k] = values + cdf_handle[k].attributes['units'] = units + + for param in params: + k = "%s_%s" % (target_slug, param) + values = [] + i = available_params.index(param) + for dkey in dkeys: + values.append(data[dkey][i]) + cdf_handle[k] = values + cdf_handle[k].attributes['units'] = PARAMETERS[param]['units'] + + k_xhee = "%s_xhee" % target_slug + k_yhee = "%s_yhee" % target_slug + values_xhee = [] + values_yhee = [] + index_x = available_params.index('xhee') + index_y = available_params.index('yhee') + for dkey in dkeys: + values_xhee.append(data[dkey][index_x]) + values_yhee.append(data[dkey][index_y]) + cdf_handle[k_xhee] = values_xhee + cdf_handle[k_yhee] = values_yhee + cdf_handle[k_xhee].attributes['units'] = 'Au' + cdf_handle[k_yhee].attributes['units'] = 'Au' + + log.debug("Writing CDF '%s'..." % cdf_filename) + cdf_handle.save(cdf_path) + + except Exception as e: + log.error("Failed to generate CDF '%s'." % cdf_filename) + removefile(cdf_path) + raise + + finally: + # cdf_handle.close() + pass + + if not isfile(cdf_path): + abort(500, "No CDF to serve. Looked at '%s'." % cdf_path) + + return send_from_directory(CACHE_DIR, cdf_filename) + + # API ######################################################################### @app.route("/cache/clear") diff --git a/web/view/home.html.jinja2 b/web/view/home.html.jinja2 index 747ef50..7b36316 100755 --- a/web/view/home.html.jinja2 +++ b/web/view/home.html.jinja2 @@ -538,8 +538,8 @@ var configuration = { orbits_container: '#orbits', api: { 'data_for_interval': "{{ request.url_root }}__.csv", - 'download': "{{ request.url_root }}__.nc", - 'samp': "{{ request.url_root }}___.nc" + 'download': "{{ request.url_root }}__.cdf", + 'samp': "{{ request.url_root }}__.cdf" }, sun: { img: '{{ static('img/sun_128.png') }}' @@ -623,6 +623,7 @@ jQuery().ready(function($){ }); $('#download').on("click", function(e){ var url = sw.buildDownloadUrl(); + console.info("Downloading " + url); $.ajax({ type: 'GET', url: url, @@ -677,7 +678,7 @@ jQuery().ready(function($){ } var name = sw.buildSampName(); var url = sw.buildSampUrl(); - console.log(name, url); + console.info("Trying to activate SAMP…", name, url); connector.runWithConnection(function (connection) { var msg = new samp.Message("table.load.cdf", { "name": name, -- libgit2 0.21.2