From 952e3d8f07394e67498b318c70f112f833568bba Mon Sep 17 00:00:00 2001 From: Goutte Date: Thu, 28 Sep 2017 21:50:49 +0200 Subject: [PATCH] Move to another spacepy for CDF generation, and clean up the python dependencies. --- requirements.txt | 47 +++++++++++++++++++++++++++++++++-------------- web/run.py | 67 ++++++++++++++++++++++++++++++++++--------------------------------- 2 files changed, 67 insertions(+), 47 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3569ecc..b35837f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,41 @@ -## The following requirements were added by pip freeze: -appdirs==1.4.3 -click==6.7 -dateutils==0.6.6 +## The following requirements were originally generated with `pip freeze`. + + +## MAIN DEPENDENCIES + Flask==0.12 -h5py==2.3.1 -itsdangerous==0.24 Jinja2==2.9.5 Markdown==2.6.8 MarkupSafe==1.0 +python-slugify==1.2.4 + + +## SECOND LEVEL DEPS +# (We could probably comment them out, but hey) + +appdirs==1.4.3 +click==6.7 +dateutils==0.6.6 +itsdangerous==0.24 mglob==0.4 -netCDF4==1.2.7 -numpy==1.12.0 -packaging==16.8 -#pkg-resources==0.0.0 pyparsing==2.2.0 -python-slugify==1.2.4 -PyYAML==3.12 +packaging==16.8 six==1.10.0 Werkzeug==0.12 -# spacepy==0.7 # requires a lot of system deps, like fortran -CDF==0.32 \ No newline at end of file + + +## FILE FORMATS + +PyYAML==3.12 +netCDF4==1.2.7 +h5py==2.3.1 +# We only use pycdf from spacepy. +spacepy==0.1.* +# spacepy does not like recent numpy versions (1.7 looks ok) +numpy==1.7.* + + +## WEIRD STUFF + +# This version number makes pip yell on the prod server (?) +#pkg-resources==0.0.0 diff --git a/web/run.py b/web/run.py index 29ff61e..4114908 100755 --- a/web/run.py +++ b/web/run.py @@ -9,7 +9,6 @@ import random import tarfile import time import urllib -import cdf from csv import writer as csv_writer from math import sqrt from os import environ, remove as removefile @@ -61,6 +60,9 @@ log.addHandler(logHandler) # HARDCODED CONFIGURATION ##################################################### +# Absolute path to the installed CDF library from https://cdf.gsfc.nasa.gov/ +CDF_LIB = '/usr/local/lib/libcdf' + # Absolute path to the data cache directory CACHE_DIR = get_path('../cache') @@ -930,13 +932,15 @@ def download_targets_cdf(targets, started_at, stopped_at): if not isfile(cdf_path): log.debug("Creating the CDF file '%s'..." % cdf_filename) + environ['CDF_LIB'] = CDF_LIB + from spacepy import pycdf try: - cdf_handle = cdf.archive() + cdf_handle = pycdf.CDF(cdf_path, masterpath='') description = "Model and orbit data for %s." % \ ', '.join([t['name'] for t in targets_configs]) - cdf_handle.attributes['Description'] = description - cdf_handle.attributes['Author'] = "Heliopropa.irap.omp.eu (CDPP)" - cdf_handle.attributes['Created'] = str(time.ctime(time.time())) + cdf_handle.attrs['Description'] = description + cdf_handle.attrs['Author'] = "Heliopropa.irap.omp.eu (CDPP)" + cdf_handle.attrs['Created'] = str(time.ctime(time.time())) available_params = list(PROPERTIES) for target in targets_configs: @@ -945,15 +949,13 @@ def download_targets_cdf(targets, started_at, stopped_at): dkeys = sorted(data) values = [] - units = "hours since 1970-01-01 00:00:00" - calendar = "standard" for dkey in dkeys: - time_as_string = data[dkey][0][:-6] # remove +00:00 tail - date = datetime.datetime.strptime(time_as_string, FILE_DATE_FMT) - values.append(date2num(date, units=units, calendar=calendar)) - k = "%s_time" % target_slug - cdf_handle[k] = values - cdf_handle[k].attributes['units'] = units + time_str = data[dkey][0][:-6] # remove +00:00 tail + date = datetime.datetime.strptime(time_str, FILE_DATE_FMT) + values.append(date) + kt = "%s_time" % target_slug + cdf_handle[kt] = values + cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D" for param in params: k = "%s_%s" % (target_slug, param) @@ -962,10 +964,16 @@ def download_targets_cdf(targets, started_at, stopped_at): for dkey in dkeys: values.append(data[dkey][i]) cdf_handle[k] = values - cdf_handle[k].attributes['units'] = PARAMETERS[param]['units'] - - k_xhee = "%s_xhee" % target_slug - k_yhee = "%s_yhee" % target_slug + attrs = cdf_handle[k].attrs + attrs['UNITS'] = PARAMETERS[param]['units'] + attrs['LABLAXIS'] = PARAMETERS[param]['name'] + attrs['FIELDNAM'] = PARAMETERS[param]['title'] + if values: + attrs['VALIDMIN'] = min(values) + attrs['VALIDMAX'] = max(values) + + kx = "%s_xhee" % target_slug + ky = "%s_yhee" % target_slug values_xhee = [] values_yhee = [] index_x = available_params.index('xhee') @@ -973,23 +981,21 @@ def download_targets_cdf(targets, started_at, stopped_at): for dkey in dkeys: values_xhee.append(data[dkey][index_x]) values_yhee.append(data[dkey][index_y]) - cdf_handle[k_xhee] = values_xhee - cdf_handle[k_yhee] = values_yhee - cdf_handle[k_xhee].attributes['units'] = 'Au' - cdf_handle[k_yhee].attributes['units'] = 'Au' + cdf_handle[kx] = values_xhee + cdf_handle[ky] = values_yhee + cdf_handle[kx].attrs['UNITS'] = 'Au' + cdf_handle[ky].attrs['UNITS'] = 'Au' log.debug("Writing CDF '%s'..." % cdf_filename) - cdf_handle.save(cdf_path) + cdf_handle.close() + log.debug("Wrote CDF '%s'." % cdf_filename) except Exception as e: log.error("Failed to generate CDF '%s'." % cdf_filename) - removefile(cdf_path) + if isfile(cdf_path): + removefile(cdf_path) raise - finally: - # cdf_handle.close() - pass - if not isfile(cdf_path): abort(500, "No CDF to serve. Looked at '%s'." % cdf_path) @@ -1080,11 +1086,6 @@ def log_clear(): # MAIN ######################################################################## if __name__ == "__main__": - - a = "2015171150000001" - - - - # Debug mode on, as the production server does not use this but run.wsgi + # Debug mode is on, as the production server does not use this but run.wsgi extra_files = [get_path('../config.yml')] app.run(debug=True, extra_files=extra_files) -- libgit2 0.21.2