From bc18b96cedce9e4dcbc8ed20a08c7888d6da2b74 Mon Sep 17 00:00:00 2001 From: Goutte Date: Tue, 15 Aug 2017 04:22:30 +0200 Subject: [PATCH] Implement first (working!) draft of NetCDF generation. Still more to do, like the time and the orbits. --- web/run.py | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 18 deletions(-) diff --git a/web/run.py b/web/run.py index 9cc15a7..37dc936 100755 --- a/web/run.py +++ b/web/run.py @@ -1,26 +1,23 @@ -import random -import datetime import StringIO -from math import sqrt - -from os import listdir, environ, remove as removefile -from os.path import isfile, join, abspath, dirname, isdir - -import csv -import json +import datetime +import time import gzip +import json +import logging +import random import tarfile import urllib -import logging -from pprint import pprint from csv import writer as csv_writer -from yaml import load as yaml_load +from math import sqrt +from os import environ, remove as removefile +from os.path import isfile, join, abspath, dirname + from flask import Flask -from flask import redirect, url_for, send_from_directory, abort as abort_flask from flask import request +from flask import url_for, send_from_directory, abort as abort_flask from jinja2 import Environment, FileSystemLoader from netCDF4 import Dataset - +from yaml import load as yaml_load # PATH RELATIVITY ############################################################# @@ -162,6 +159,13 @@ def render_view(view, context=None): # ) +def is_list_in_list(needle, haystack): + for n in needle: + if n not in haystack: + return False + return True + + def datetime_from_list(time_list): """ Datetimes in retrieved CDFs are stored as lists of numbers, @@ -276,9 +280,26 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): # The order matters. If you change this you also need to change the # innermost loop of `get_data_for_target`. # The javascript knows the targets' properties under these names. +# DEPRECATED, use PARAMETERS instead PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens', 'angl', 'xhci', 'yhci') +PARAMETERS = { + 'vlen': { + 'slug': 'vlen', + 'name': 'Velocity', + 'unit': 'km/s', + 'show': True, + }, + 'magn': { + 'slug': 'magn', + 'name': 'Magnetism', + 'unit': 'nT', + 'show': True, + }, + # fixme: add more +} + def get_data_for_target(target_config, started_at, stopped_at): """ @@ -509,12 +530,11 @@ def download_target_csv(target, started_at, stopped_at): @app.route("/__.tar.gz") def download_targets_tarball(targets, started_at, stopped_at): """ - Grab data and orbit data for the specified `target`, - rearrange it and return it as a CSV file. - `started_at` and `stopped_at` should be UTC. + Grab data and orbit data for each of the specified `targets`, + in their own CSV file, and make a tarball of them. + `started_at` and `stopped_at` should be UTC strings. targets: string list of targets' slugs, separated by `-`. - This will fail hard if targets' slugs start having `-` in them. """ separator = '-' targets = targets.split(separator) @@ -566,6 +586,93 @@ def download_targets_tarball(targets, started_at, stopped_at): return send_from_directory(get_path("../cache/"), gzip_filename) +@app.route("/___.nc") +def download_targets_netcdf(targets, params, started_at, stopped_at): + """ + Grab data and orbit data for the specified `target`, + rearrange it and return it as a CSV file. + `started_at` and `stopped_at` should be UTC. + + targets: string list of targets' slugs, separated by `-`. + params: string list of targets' parameters, separated by `-`. + """ + separator = '-' + targets = targets.split(separator) + targets.sort() + targets_configs = [] + for target in targets: + if not target: + abort(400, "Invalid targets format : `%s`." % targets) + targets_configs.append(get_target_config(target)) + if 0 == len(targets_configs): + abort(400, "No valid targets specified. What are you doing?") + params = params.split(separator) + params.sort() + if 0 == len(params): + abort(400, "No valid parameters specified. What are you doing?") + if not is_list_in_list(params, PARAMETERS.keys()): + abort(400, "Some parameters are not recognized in '%s'." % str(params)) + + date_fmt = "%Y-%m-%dT%H:%M:%S" + try: + started_at = datetime.datetime.strptime(started_at, date_fmt) + except: + abort(400, "Invalid started_at parameter : '%s'." % started_at) + try: + stopped_at = datetime.datetime.strptime(stopped_at, date_fmt) + except: + abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) + sta = started_at.strftime(date_fmt) + sto = stopped_at.strftime(date_fmt) + + nc_filename = "%s_%s_%s_%s.nc" % \ + (separator.join(targets), separator.join(params), sta, sto) + nc_path = get_path("../cache/%s" % nc_filename) + + if not isfile(nc_path): + log.debug("Creating the NetCDF file '%s'..." % nc_filename) + nc_handle = Dataset(nc_path, "w", format="NETCDF4") + try: + nc_handle.description = "TODO" # todo + nc_handle.history = "Created " + time.ctime(time.time()) + nc_handle.source = "Transplanet (CDDP)" + available_params = list(PROPERTIES) + for target in targets_configs: + target_slug = target['slug'] + log.debug("Adding group '%s' to the NetCDF..." % target_slug) + nc_group = nc_handle.createGroup(target_slug) + data = get_data_for_target(target, started_at, stopped_at) + dkeys = sorted(data) + nc_handle.createDimension('dim_'+target_slug, len(dkeys)) + # todo: add time + nc_vars = [] + indices = [] + for param in params: + indices.append(available_params.index(param)) + nc_var = nc_group.createVariable( + param, 'f8', ('dim_'+target_slug,) + ) + nc_var.units = PARAMETERS[param]['unit'] + nc_vars.append(nc_var) + for i, nc_var in enumerate(nc_vars): + index = indices[i] + values = [] + for dkey in dkeys: + dval = data[dkey] + values.append(dval[index]) + nc_var[:] = values + # todo: add orbit x and y + except Exception as e: + raise e + finally: + nc_handle.close() + + if not isfile(nc_path): + abort(500, "No NetCDF to serve. Looked at '%s'." % nc_path) + + return send_from_directory(get_path("../cache/"), nc_filename) + + # API ######################################################################### @app.route("/cache/clear") -- libgit2 0.21.2