From 4aaf6874ad40138d122eee284abfc6b8f4579af5 Mon Sep 17 00:00:00 2001 From: Goutte Date: Mon, 26 Mar 2018 07:32:17 +0200 Subject: [PATCH] Try fixing the generated CDF file, and make sure we update spacepy's toolbox lazily. --- web/run.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/web/run.py b/web/run.py index a889b58..2ce937b 100755 --- a/web/run.py +++ b/web/run.py @@ -727,6 +727,21 @@ def increment_hit_counter(): return hit_count +def update_spacepy(): + """ + Importing pydcf will fail if the toolbox is not up to date. + """ + environ['SPACEPY'] = CACHE_DIR + environ['CDF_LIB'] = CDF_LIB + try: + log.info("Updating spacepy's toolbox…") + import spacepy.toolbox + + spacepy.toolbox.update() + except Exception as e: + log.error("Failed to update spacepy : %s." % e) + + tpl_global_vars['visits'] = get_hit_counter() @@ -866,6 +881,8 @@ def download_targets_tarball(targets, inp, started_at, stopped_at): @app.route("/____.nc") def download_targets_netcdf(targets, inp, params, started_at, stopped_at): """ + NOTE : This is not used anymore. + Grab data and orbit data for the specified `target`, rearrange it and return it as a NetCDF file. `started_at` and `stopped_at` are expected to be UTC. @@ -883,6 +900,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at): targets_configs.append(get_target_config(target)) if 0 == len(targets_configs): abort(400, "No valid targets specified. What are you doing?") + params = params.split(separator) params.sort() if 0 == len(params): @@ -1009,12 +1027,6 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): abort(400, "No valid targets specified. What are you doing?") params = PARAMETERS.keys() - # params = params.split(separator) - # params.sort() - # if 0 == len(params): - # abort(400, "No valid parameters specified. What are you doing?") - # if not is_list_in_list(params, PARAMETERS.keys()): - # abort(400, "Some parameters are not recognized in '%s'." % str(params)) try: started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) @@ -1036,21 +1048,18 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): if not isfile(cdf_path): log.debug("Creating the CDF file '%s'..." % cdf_filename) - environ['SPACEPY'] = CACHE_DIR - environ['CDF_LIB'] = CDF_LIB - try: - # Importing pydcf will fail if the toolbox is not up to date. - # todo: maybe move this to the daily cache warmup ? - log.info("Updating spacepy's toolbox…") - import spacepy.toolbox - spacepy.toolbox.update() - except Exception as e: - log.error("Failed to update spacepy : %s." % e) try: from spacepy import pycdf - except ImportError as e: - log.error("Failed to import pycdf from spacepy : %s" % e) - raise + except ImportError: + # If spacepy's toolbox is not up-to-date, importing will fail. + # So, let's update and try again ! + update_spacepy() + try: + from spacepy import pycdf + except ImportError as e: + log.error("Failed to import pycdf from spacepy : %s" % e) + raise + try: cdf_handle = pycdf.CDF(cdf_path, masterpath='') description = "Model and orbit data for %s." % \ @@ -1074,15 +1083,26 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): date = datetime.datetime.strptime(time_str, FILE_DATE_FMT) values.append(date) kt = "%s_time" % target_slug + cdf_handle.new(kt, type=pycdf.const.CDF_EPOCH) cdf_handle[kt] = values - cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D" + # cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D" for param in params: k = "%s_%s" % (target_slug, param) + # print("PARAM %s" % k) values = [] i = available_params.index(param) + has_nones = False for dkey in dkeys: - values.append(data[dkey][i]) + value = data[dkey][i] + if value is None: + has_nones = True + values.append(value) + if has_nones: + # PyCDF hates it when there are Nones. + # Since we don't know what value to set instead, + # let's skip the param altogether. + continue cdf_handle[k] = values attrs = cdf_handle[k].attrs attrs['UNITS'] = PARAMETERS[param]['units'] @@ -1102,11 +1122,14 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): value_xhee = data[dkey][index_x] value_yhee = data[dkey][index_y] # We've got some `None`s cropping up in the data sometimes. + # PyCDF does not digest Nones at all. # While they solve this upstream, let's make an ugly fix! if (value_xhee is not None) and (value_yhee is not None): values_xhee.append(value_xhee) values_yhee.append(value_yhee) else: + values_xhee.append(0) + values_yhee.append(0) log.warn("Orbit data for %s has NaNs." % target_slug) cdf_handle[kx] = values_xhee cdf_handle[ky] = values_yhee @@ -1174,10 +1197,12 @@ def cache_warmup(): inp = 'l1' # default input, maybe warm them all up ? targets = get_active_targets() - for target in targets: - download_target_csv(target['slug'], inp, sta, sto) targets_slugs = [target['slug'] for target in targets] targets_slugs.sort() + + update_spacepy() + for target in targets: + download_target_csv(target['slug'], inp, sta, sto) download_targets_cdf('-'.join(targets_slugs), inp, sta, sto) warmup_ended_at = datetime.datetime.now() -- libgit2 0.21.2