Try fixing the generated CDF file, and make sure we update spacepy's toolbox lazily.

Goutte
1 parent dd8621f8
Showing 1 changed file with 48 additions and 23 deletions Show diff stats
web/run.py
@@ -727,6 +727,21 @@ def increment_hit_counter():
     return hit_count
  
  
+def update_spacepy():
+    """
+    Importing pydcf will fail if the toolbox is not up to date.
+    """
+    environ['SPACEPY'] = CACHE_DIR
+    environ['CDF_LIB'] = CDF_LIB
+    try:
+        log.info("Updating spacepy's toolbox…")
+        import spacepy.toolbox
+
+        spacepy.toolbox.update()
+    except Exception as e:
+        log.error("Failed to update spacepy : %s." % e)
+
+
 tpl_global_vars['visits'] = get_hit_counter()
  
  
@@ -866,6 +881,8 @@ def download_targets_tarball(targets, inp, started_at, stopped_at):
 @app.route("/<targets>_<inp>_<params>_<started_at>_<stopped_at>.nc")
 def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
     """
+    NOTE : This is not used anymore.
+    
     Grab data and orbit data for the specified `target`,
     rearrange it and return it as a NetCDF file.
     `started_at` and `stopped_at` are expected to be UTC.
@@ -883,6 +900,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
         targets_configs.append(get_target_config(target))
     if 0 == len(targets_configs):
         abort(400, "No valid targets specified. What are you doing?")
+
     params = params.split(separator)
     params.sort()
     if 0 == len(params):
@@ -1009,12 +1027,6 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
         abort(400, "No valid targets specified. What are you doing?")
  
     params = PARAMETERS.keys()
-    # params = params.split(separator)
-    # params.sort()
-    # if 0 == len(params):
-    #     abort(400, "No valid parameters specified. What are you doing?")
-    # if not is_list_in_list(params, PARAMETERS.keys()):
-    #     abort(400, "Some parameters are not recognized in '%s'." % str(params))
  
     try:
         started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
@@ -1036,21 +1048,18 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
  
     if not isfile(cdf_path):
         log.debug("Creating the CDF file '%s'..." % cdf_filename)
-        environ['SPACEPY'] = CACHE_DIR
-        environ['CDF_LIB'] = CDF_LIB
-        try:
-            # Importing pydcf will fail if the toolbox is not up to date.
-            # todo: maybe move this to the daily cache warmup ?
-            log.info("Updating spacepy's toolbox…")
-            import spacepy.toolbox
-            spacepy.toolbox.update()
-        except Exception as e:
-            log.error("Failed to update spacepy : %s." % e)
         try:
             from spacepy import pycdf
-        except ImportError as e:
-            log.error("Failed to import pycdf from spacepy : %s" % e)
-            raise
+        except ImportError:
+            # If spacepy's toolbox is not up-to-date, importing will fail.
+            # So, let's update and try again !
+            update_spacepy()
+            try:
+                from spacepy import pycdf
+            except ImportError as e:
+                log.error("Failed to import pycdf from spacepy : %s" % e)
+                raise
+
         try:
             cdf_handle = pycdf.CDF(cdf_path, masterpath='')
             description = "Model and orbit data for %s." % \
@@ -1074,15 +1083,26 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
                     date = datetime.datetime.strptime(time_str, FILE_DATE_FMT)
                     values.append(date)
                 kt = "%s_time" % target_slug
+                cdf_handle.new(kt, type=pycdf.const.CDF_EPOCH)
                 cdf_handle[kt] = values
-                cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D"
+                # cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D"
  
                 for param in params:
                     k = "%s_%s" % (target_slug, param)
+                    # print("PARAM %s" % k)
                     values = []
                     i = available_params.index(param)
+                    has_nones = False
                     for dkey in dkeys:
-                        values.append(data[dkey][i])
+                        value = data[dkey][i]
+                        if value is None:
+                            has_nones = True
+                        values.append(value)
+                    if has_nones:
+                        # PyCDF hates it when there are Nones.
+                        # Since we don't know what value to set instead,
+                        # let's skip the param altogether.
+                        continue
                     cdf_handle[k] = values
                     attrs = cdf_handle[k].attrs
                     attrs['UNITS'] = PARAMETERS[param]['units']
@@ -1102,11 +1122,14 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
                     value_xhee = data[dkey][index_x]
                     value_yhee = data[dkey][index_y]
                     # We've got some `None`s cropping up in the data sometimes.
+                    # PyCDF does not digest Nones at all.
                     # While they solve this upstream, let's make an ugly fix!
                     if (value_xhee is not None) and (value_yhee is not None):
                         values_xhee.append(value_xhee)
                         values_yhee.append(value_yhee)
                     else:
+                        values_xhee.append(0)
+                        values_yhee.append(0)
                         log.warn("Orbit data for %s has NaNs." % target_slug)
                 cdf_handle[kx] = values_xhee
                 cdf_handle[ky] = values_yhee
@@ -1174,10 +1197,12 @@ def cache_warmup():
     inp = 'l1'  # default input, maybe warm them all up ?
  
     targets = get_active_targets()
-    for target in targets:
-        download_target_csv(target['slug'], inp, sta, sto)
     targets_slugs = [target['slug'] for target in targets]
     targets_slugs.sort()
+
+    update_spacepy()
+    for target in targets:
+        download_target_csv(target['slug'], inp, sta, sto)
     download_targets_cdf('-'.join(targets_slugs), inp, sta, sto)
  
     warmup_ended_at = datetime.datetime.now()