Generate a CDF file (not NetCDF)

Bump to v1.0.0-rc4

Generate a CDF file (not NetCDF)
Bump to v1.0.0-rc4
Goutte
1 parent 7b5642ae
Showing 5 changed files with 124 additions and 9 deletions Show diff stats
CHANGELOG.md
VERSION
requirements.txt
web/run.py
web/view/home.html.jinja2
@@ -7,8 +7,6 @@
 - [ ] CRON statements to call the cache cleanup and warmup
 - [ ] Cache warmup (generate for today's default interval) `/cache/warmup`
 - [ ] Give the future data another color
-- [ ] Sort times series by closeness to the sun
-- [ ] Generate a CDF file (not NetCDF)
 An heliospheric propagation 1D MHD model for solar wind prediction at planets, probes and comets.
@@ -16,6 +14,8 @@ An heliospheric propagation 1D MHD model for solar wind prediction at planets, p
 ## 1.0.0-rc4
+- [x] Sort times series by closeness to the sun
+- [x] Generate a CDF file (not NetCDF)
 - [x] Normalize time interval for time series
 - [x] Make the footer images clickable
 - [x] Highlight the visits counter
-1.0.0-rc3
 \ No newline at end of file
+1.0.0-rc4
 \ No newline at end of file
@@ -18,3 +18,5 @@ python-slugify==1.2.4
 PyYAML==3.12
 six==1.10.0
 Werkzeug==0.12
+# spacepy==0.7  # requires a lot of system deps, like fortran
+CDF==0.32
 \ No newline at end of file
@@ -9,6 +9,7 @@ import random
 import tarfile
 import time
 import urllib
+import cdf
 from csv import writer as csv_writer
 from math import sqrt
 from os import environ, remove as removefile
@@ -112,7 +113,7 @@ PARAMETERS = {
         'slug': 'dens',
         'name': 'Density',
         'title': 'The density N.',
-        'units': u'cm⁻³',
+        'units': 'cm^-3',
         'active': False,
         'position': 50,
     },
@@ -438,7 +439,7 @@ def get_data_for_target(target_config, started_at, stopped_at):
             _s1 = min(_s1, _sto)
         else:
             _s1 = _sto
-        return _s1, _s0
+        return _s0, _s1
     precision = "%Y-%m-%dT%H"  # model and orbits times are only equal-ish
     orbit_data = {}  # keys are datetime as str, values arrays of XY
@@ -770,7 +771,7 @@ def download_targets_tarball(targets, started_at, stopped_at):
 def download_targets_netcdf(targets, params, started_at, stopped_at):
     """
     Grab data and orbit data for the specified `target`,
-    rearrange it and return it as a CSV file.
+    rearrange it and return it as a NetCDF file.
     `started_at` and `stopped_at` are expected to be UTC.
     targets: string list of targets' slugs, separated by `-`.
@@ -884,6 +885,117 @@ def download_targets_netcdf(targets, params, started_at, stopped_at):
     return send_from_directory(CACHE_DIR, nc_filename)
+@app.route("/<targets>_<started_at>_<stopped_at>.cdf")
+def download_targets_cdf(targets, started_at, stopped_at):
+    """
+    Grab data and orbit data for the specified `target`,
+    rearrange it and return it as a CDF file.
+    `started_at` and `stopped_at` are expected to be UTC.
+
+    targets: string list of targets' slugs, separated by `-`.
+    params: string list of targets' parameters, separated by `-`.
+    """
+    separator = '-'  # /!\ this char should never be in target's slugs
+    targets = targets.split(separator)
+    targets.sort()
+    targets_configs = []
+    for target in targets:
+        if not target:
+            abort(400, "Invalid targets format : `%s`." % targets)
+        targets_configs.append(get_target_config(target))
+    if 0 == len(targets_configs):
+        abort(400, "No valid targets specified. What are you doing?")
+
+    params = PARAMETERS.keys()
+    # params = params.split(separator)
+    # params.sort()
+    # if 0 == len(params):
+    #     abort(400, "No valid parameters specified. What are you doing?")
+    # if not is_list_in_list(params, PARAMETERS.keys()):
+    #     abort(400, "Some parameters are not recognized in '%s'." % str(params))
+
+    try:
+        started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
+    except:
+        abort(400, "Invalid started_at parameter : '%s'." % started_at)
+    try:
+        stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT)
+    except:
+        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
+    sta = started_at.strftime(FILE_DATE_FMT)
+    sto = stopped_at.strftime(FILE_DATE_FMT)
+
+    cdf_filename = "%s_%s_%s.cdf" % (separator.join(targets), sta, sto)
+    cdf_path = join(CACHE_DIR, cdf_filename)
+
+    if not isfile(cdf_path):
+        log.debug("Creating the CDF file '%s'..." % cdf_filename)
+        try:
+            cdf_handle = cdf.archive()
+            description = "Model and orbit data for %s." % \
+                ', '.join([t['name'] for t in targets_configs])
+            cdf_handle.attributes['Description'] = description
+            cdf_handle.attributes['Author'] = "Heliopropa.irap.omp.eu (CDPP)"
+            cdf_handle.attributes['Created'] = str(time.ctime(time.time()))
+
+            available_params = list(PROPERTIES)
+            for target in targets_configs:
+                target_slug = target['slug']
+                data = get_data_for_target(target, started_at, stopped_at)
+                dkeys = sorted(data)
+
+                values = []
+                units = "hours since 1970-01-01 00:00:00"
+                calendar = "standard"
+                for dkey in dkeys:
+                    time_as_string = data[dkey][0][:-6]  # remove +00:00 tail
+                    date = datetime.datetime.strptime(time_as_string, FILE_DATE_FMT)
+                    values.append(date2num(date, units=units, calendar=calendar))
+                k = "%s_time" % target_slug
+                cdf_handle[k] = values
+                cdf_handle[k].attributes['units'] = units
+
+                for param in params:
+                    k = "%s_%s" % (target_slug, param)
+                    values = []
+                    i = available_params.index(param)
+                    for dkey in dkeys:
+                        values.append(data[dkey][i])
+                    cdf_handle[k] = values
+                    cdf_handle[k].attributes['units'] = PARAMETERS[param]['units']
+
+                k_xhee = "%s_xhee" % target_slug
+                k_yhee = "%s_yhee" % target_slug
+                values_xhee = []
+                values_yhee = []
+                index_x = available_params.index('xhee')
+                index_y = available_params.index('yhee')
+                for dkey in dkeys:
+                    values_xhee.append(data[dkey][index_x])
+                    values_yhee.append(data[dkey][index_y])
+                cdf_handle[k_xhee] = values_xhee
+                cdf_handle[k_yhee] = values_yhee
+                cdf_handle[k_xhee].attributes['units'] = 'Au'
+                cdf_handle[k_yhee].attributes['units'] = 'Au'
+
+            log.debug("Writing CDF '%s'..." % cdf_filename)
+            cdf_handle.save(cdf_path)
+
+        except Exception as e:
+            log.error("Failed to generate CDF '%s'." % cdf_filename)
+            removefile(cdf_path)
+            raise
+
+        finally:
+            # cdf_handle.close()
+            pass
+
+    if not isfile(cdf_path):
+        abort(500, "No CDF to serve. Looked at '%s'." % cdf_path)
+
+    return send_from_directory(CACHE_DIR, cdf_filename)
+
+
 # API #########################################################################
 @app.route("/cache/clear")
@@ -538,8 +538,8 @@ var configuration = {
   orbits_container: '#orbits',
   api: {
       'data_for_interval': "{{ request.url_root }}<target>_<started_at>_<stopped_at>.csv",
-      'download': "{{ request.url_root }}<targets>_<started_at>_<stopped_at>.nc",
-      'samp': "{{ request.url_root }}<targets>_<params>_<started_at>_<stopped_at>.nc"
+      'download': "{{ request.url_root }}<targets>_<started_at>_<stopped_at>.cdf",
+      'samp': "{{ request.url_root }}<targets>_<started_at>_<stopped_at>.cdf"
   },
   sun: {
     img: '{{ static('img/sun_128.png') }}'
@@ -623,6 +623,7 @@ jQuery().ready(function($){
   });
   $('#download').on("click", function(e){
     var url = sw.buildDownloadUrl();
+    console.info("Downloading " + url);
     $.ajax({
       type: 'GET',
       url: url,
@@ -677,7 +678,7 @@ jQuery().ready(function($){
       }
       var name = sw.buildSampName();
       var url = sw.buildSampUrl();
-      console.log(name, url);
+      console.info("Trying to activate SAMP…", name, url);
       connector.runWithConnection(function (connection) {
         var msg = new samp.Message("table.load.cdf", {
           "name": name,