Move to another spacepy for CDF generation, and clean up the python dependencies.

Goutte
1 parent 0af09c17
Showing 2 changed files with 67 additions and 47 deletions Show diff stats
requirements.txt
web/run.py
-## The following requirements were added by pip freeze:
-appdirs==1.4.3
-click==6.7
-dateutils==0.6.6
+## The following requirements were originally generated with `pip freeze`.
+
+
+## MAIN DEPENDENCIES
+
 Flask==0.12
-h5py==2.3.1
-itsdangerous==0.24
 Jinja2==2.9.5
 Markdown==2.6.8
 MarkupSafe==1.0
+python-slugify==1.2.4
+
+
+## SECOND LEVEL DEPS
+# (We could probably comment them out, but hey)
+
+appdirs==1.4.3
+click==6.7
+dateutils==0.6.6
+itsdangerous==0.24
 mglob==0.4
-netCDF4==1.2.7
-numpy==1.12.0
-packaging==16.8
-#pkg-resources==0.0.0
 pyparsing==2.2.0
-python-slugify==1.2.4
-PyYAML==3.12
+packaging==16.8
 six==1.10.0
 Werkzeug==0.12
-# spacepy==0.7  # requires a lot of system deps, like fortran
-CDF==0.32
 \ No newline at end of file
+
+
+## FILE FORMATS
+
+PyYAML==3.12
+netCDF4==1.2.7
+h5py==2.3.1
+# We only use pycdf from spacepy.
+spacepy==0.1.*
+# spacepy does not like recent numpy versions (1.7 looks ok)
+numpy==1.7.*
+
+
+## WEIRD STUFF
+
+# This version number makes pip yell on the prod server (?)
+#pkg-resources==0.0.0
@@ -9,7 +9,6 @@ import random
 import tarfile
 import time
 import urllib
-import cdf
 from csv import writer as csv_writer
 from math import sqrt
 from os import environ, remove as removefile
@@ -61,6 +60,9 @@ log.addHandler(logHandler)
 # HARDCODED CONFIGURATION #####################################################
+# Absolute path to the installed CDF library from https://cdf.gsfc.nasa.gov/
+CDF_LIB = '/usr/local/lib/libcdf'
+
 # Absolute path to the data cache directory
 CACHE_DIR = get_path('../cache')
@@ -930,13 +932,15 @@ def download_targets_cdf(targets, started_at, stopped_at):
     if not isfile(cdf_path):
         log.debug("Creating the CDF file '%s'..." % cdf_filename)
+        environ['CDF_LIB'] = CDF_LIB
+        from spacepy import pycdf
         try:
-            cdf_handle = cdf.archive()
+            cdf_handle = pycdf.CDF(cdf_path, masterpath='')
             description = "Model and orbit data for %s." % \
                 ', '.join([t['name'] for t in targets_configs])
-            cdf_handle.attributes['Description'] = description
-            cdf_handle.attributes['Author'] = "Heliopropa.irap.omp.eu (CDPP)"
-            cdf_handle.attributes['Created'] = str(time.ctime(time.time()))
+            cdf_handle.attrs['Description'] = description
+            cdf_handle.attrs['Author'] = "Heliopropa.irap.omp.eu (CDPP)"
+            cdf_handle.attrs['Created'] = str(time.ctime(time.time()))
             available_params = list(PROPERTIES)
             for target in targets_configs:
@@ -945,15 +949,13 @@ def download_targets_cdf(targets, started_at, stopped_at):
                 dkeys = sorted(data)
                 values = []
-                units = "hours since 1970-01-01 00:00:00"
-                calendar = "standard"
                 for dkey in dkeys:
-                    time_as_string = data[dkey][0][:-6]  # remove +00:00 tail
-                    date = datetime.datetime.strptime(time_as_string, FILE_DATE_FMT)
-                    values.append(date2num(date, units=units, calendar=calendar))
-                k = "%s_time" % target_slug
-                cdf_handle[k] = values
-                cdf_handle[k].attributes['units'] = units
+                    time_str = data[dkey][0][:-6]  # remove +00:00 tail
+                    date = datetime.datetime.strptime(time_str, FILE_DATE_FMT)
+                    values.append(date)
+                kt = "%s_time" % target_slug
+                cdf_handle[kt] = values
+                cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D"
                 for param in params:
                     k = "%s_%s" % (target_slug, param)
@@ -962,10 +964,16 @@ def download_targets_cdf(targets, started_at, stopped_at):
                     for dkey in dkeys:
                         values.append(data[dkey][i])
                     cdf_handle[k] = values
-                    cdf_handle[k].attributes['units'] = PARAMETERS[param]['units']
-
-                k_xhee = "%s_xhee" % target_slug
-                k_yhee = "%s_yhee" % target_slug
+                    attrs = cdf_handle[k].attrs
+                    attrs['UNITS'] = PARAMETERS[param]['units']
+                    attrs['LABLAXIS'] = PARAMETERS[param]['name']
+                    attrs['FIELDNAM'] = PARAMETERS[param]['title']
+                    if values:
+                        attrs['VALIDMIN'] = min(values)
+                        attrs['VALIDMAX'] = max(values)
+
+                kx = "%s_xhee" % target_slug
+                ky = "%s_yhee" % target_slug
                 values_xhee = []
                 values_yhee = []
                 index_x = available_params.index('xhee')
@@ -973,23 +981,21 @@ def download_targets_cdf(targets, started_at, stopped_at):
                 for dkey in dkeys:
                     values_xhee.append(data[dkey][index_x])
                     values_yhee.append(data[dkey][index_y])
-                cdf_handle[k_xhee] = values_xhee
-                cdf_handle[k_yhee] = values_yhee
-                cdf_handle[k_xhee].attributes['units'] = 'Au'
-                cdf_handle[k_yhee].attributes['units'] = 'Au'
+                cdf_handle[kx] = values_xhee
+                cdf_handle[ky] = values_yhee
+                cdf_handle[kx].attrs['UNITS'] = 'Au'
+                cdf_handle[ky].attrs['UNITS'] = 'Au'
             log.debug("Writing CDF '%s'..." % cdf_filename)
-            cdf_handle.save(cdf_path)
+            cdf_handle.close()
+            log.debug("Wrote CDF '%s'." % cdf_filename)
         except Exception as e:
             log.error("Failed to generate CDF '%s'." % cdf_filename)
-            removefile(cdf_path)
+            if isfile(cdf_path):
+                removefile(cdf_path)
             raise
-        finally:
-            # cdf_handle.close()
-            pass
-
     if not isfile(cdf_path):
         abort(500, "No CDF to serve. Looked at '%s'." % cdf_path)
@@ -1080,11 +1086,6 @@ def log_clear():
 # MAIN ########################################################################
 if __name__ == "__main__":
-
-    a = "2015171150000001"
-
-
-
-    # Debug mode on, as the production server does not use this but run.wsgi
+    # Debug mode is on, as the production server does not use this but run.wsgi
     extra_files = [get_path('../config.yml')]
     app.run(debug=True, extra_files=extra_files)