Implement first (working!) draft of NetCDF generation.

Still more to do, like the time and the orbits.

Implement first (working!) draft of NetCDF generation.
Still more to do, like the time and the orbits.
Goutte
1 parent 3f0d5ea2
Showing 1 changed file with 125 additions and 18 deletions Show diff stats
web/run.py
-import random
-import datetime
 import StringIO
-from math import sqrt
-
-from os import listdir, environ, remove as removefile
-from os.path import isfile, join, abspath, dirname, isdir
-
-import csv
-import json
+import datetime
+import time
 import gzip
+import json
+import logging
+import random
 import tarfile
 import urllib
-import logging
-from pprint import pprint
 from csv import writer as csv_writer
-from yaml import load as yaml_load
+from math import sqrt
+from os import environ, remove as removefile
+from os.path import isfile, join, abspath, dirname
+
 from flask import Flask
-from flask import redirect, url_for, send_from_directory, abort as abort_flask
 from flask import request
+from flask import url_for, send_from_directory, abort as abort_flask
 from jinja2 import Environment, FileSystemLoader
 from netCDF4 import Dataset
-
+from yaml import load as yaml_load
 # PATH RELATIVITY #############################################################
@@ -162,6 +159,13 @@ def render_view(view, context=None):
 #     )
+def is_list_in_list(needle, haystack):
+    for n in needle:
+        if n not in haystack:
+            return False
+    return True
+
+
 def datetime_from_list(time_list):
     """
     Datetimes in retrieved CDFs are stored as lists of numbers,
@@ -276,9 +280,26 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
 # The order matters. If you change this you also need to change the
 # innermost loop of `get_data_for_target`.
 # The javascript knows the targets' properties under these names.
+# DEPRECATED, use PARAMETERS instead
 PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens',
               'angl', 'xhci', 'yhci')
+PARAMETERS = {
+    'vlen': {
+        'slug': 'vlen',
+        'name': 'Velocity',
+        'unit': 'km/s',
+        'show': True,
+    },
+    'magn': {
+        'slug': 'magn',
+        'name': 'Magnetism',
+        'unit': 'nT',
+        'show': True,
+    },
+    # fixme: add more
+}
+
 def get_data_for_target(target_config, started_at, stopped_at):
     """
@@ -509,12 +530,11 @@ def download_target_csv(target, started_at, stopped_at):
 @app.route("/<targets>_<started_at>_<stopped_at>.tar.gz")
 def download_targets_tarball(targets, started_at, stopped_at):
     """
-    Grab data and orbit data for the specified `target`,
-    rearrange it and return it as a CSV file.
-    `started_at` and `stopped_at` should be UTC.
+    Grab data and orbit data for each of the specified `targets`,
+    in their own CSV file, and make a tarball of them.
+    `started_at` and `stopped_at` should be UTC strings.
     targets: string list of targets' slugs, separated by `-`.
-    This will fail hard if targets' slugs start having `-` in them.
     """
     separator = '-'
     targets = targets.split(separator)
@@ -566,6 +586,93 @@ def download_targets_tarball(targets, started_at, stopped_at):
     return send_from_directory(get_path("../cache/"), gzip_filename)
+@app.route("/<targets>_<params>_<started_at>_<stopped_at>.nc")
+def download_targets_netcdf(targets, params, started_at, stopped_at):
+    """
+    Grab data and orbit data for the specified `target`,
+    rearrange it and return it as a CSV file.
+    `started_at` and `stopped_at` should be UTC.
+
+    targets: string list of targets' slugs, separated by `-`.
+    params: string list of targets' parameters, separated by `-`.
+    """
+    separator = '-'
+    targets = targets.split(separator)
+    targets.sort()
+    targets_configs = []
+    for target in targets:
+        if not target:
+            abort(400, "Invalid targets format : `%s`." % targets)
+        targets_configs.append(get_target_config(target))
+    if 0 == len(targets_configs):
+        abort(400, "No valid targets specified. What are you doing?")
+    params = params.split(separator)
+    params.sort()
+    if 0 == len(params):
+        abort(400, "No valid parameters specified. What are you doing?")
+    if not is_list_in_list(params, PARAMETERS.keys()):
+        abort(400, "Some parameters are not recognized in '%s'." % str(params))
+
+    date_fmt = "%Y-%m-%dT%H:%M:%S"
+    try:
+        started_at = datetime.datetime.strptime(started_at, date_fmt)
+    except:
+        abort(400, "Invalid started_at parameter : '%s'." % started_at)
+    try:
+        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
+    except:
+        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
+    sta = started_at.strftime(date_fmt)
+    sto = stopped_at.strftime(date_fmt)
+
+    nc_filename = "%s_%s_%s_%s.nc" % \
+                  (separator.join(targets), separator.join(params), sta, sto)
+    nc_path = get_path("../cache/%s" % nc_filename)
+
+    if not isfile(nc_path):
+        log.debug("Creating the NetCDF file '%s'..." % nc_filename)
+        nc_handle = Dataset(nc_path, "w", format="NETCDF4")
+        try:
+            nc_handle.description = "TODO"  # todo
+            nc_handle.history = "Created " + time.ctime(time.time())
+            nc_handle.source = "Transplanet (CDDP)"
+            available_params = list(PROPERTIES)
+            for target in targets_configs:
+                target_slug = target['slug']
+                log.debug("Adding group '%s' to the NetCDF..." % target_slug)
+                nc_group = nc_handle.createGroup(target_slug)
+                data = get_data_for_target(target, started_at, stopped_at)
+                dkeys = sorted(data)
+                nc_handle.createDimension('dim_'+target_slug, len(dkeys))
+                # todo: add time
+                nc_vars = []
+                indices = []
+                for param in params:
+                    indices.append(available_params.index(param))
+                    nc_var = nc_group.createVariable(
+                        param, 'f8', ('dim_'+target_slug,)
+                    )
+                    nc_var.units = PARAMETERS[param]['unit']
+                    nc_vars.append(nc_var)
+                for i, nc_var in enumerate(nc_vars):
+                    index = indices[i]
+                    values = []
+                    for dkey in dkeys:
+                        dval = data[dkey]
+                        values.append(dval[index])
+                    nc_var[:] = values
+                # todo: add orbit x and y
+        except Exception as e:
+            raise e
+        finally:
+            nc_handle.close()
+
+    if not isfile(nc_path):
+        abort(500, "No NetCDF to serve. Looked at '%s'." % nc_path)
+
+    return send_from_directory(get_path("../cache/"), nc_filename)
+
+
 # API #########################################################################
 @app.route("/cache/clear")