Commit 4aaf6874ad40138d122eee284abfc6b8f4579af5

Authored by Goutte
1 parent dd8621f8

Try fixing the generated CDF file, and make sure we update spacepy's toolbox lazily.

Showing 1 changed file with 48 additions and 23 deletions   Show diff stats
@@ -727,6 +727,21 @@ def increment_hit_counter(): @@ -727,6 +727,21 @@ def increment_hit_counter():
727 return hit_count 727 return hit_count
728 728
729 729
  730 +def update_spacepy():
  731 + """
  732 + Importing pydcf will fail if the toolbox is not up to date.
  733 + """
  734 + environ['SPACEPY'] = CACHE_DIR
  735 + environ['CDF_LIB'] = CDF_LIB
  736 + try:
  737 + log.info("Updating spacepy's toolbox…")
  738 + import spacepy.toolbox
  739 +
  740 + spacepy.toolbox.update()
  741 + except Exception as e:
  742 + log.error("Failed to update spacepy : %s." % e)
  743 +
  744 +
730 tpl_global_vars['visits'] = get_hit_counter() 745 tpl_global_vars['visits'] = get_hit_counter()
731 746
732 747
@@ -866,6 +881,8 @@ def download_targets_tarball(targets, inp, started_at, stopped_at): @@ -866,6 +881,8 @@ def download_targets_tarball(targets, inp, started_at, stopped_at):
866 @app.route("/<targets>_<inp>_<params>_<started_at>_<stopped_at>.nc") 881 @app.route("/<targets>_<inp>_<params>_<started_at>_<stopped_at>.nc")
867 def download_targets_netcdf(targets, inp, params, started_at, stopped_at): 882 def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
868 """ 883 """
  884 + NOTE : This is not used anymore.
  885 +
869 Grab data and orbit data for the specified `target`, 886 Grab data and orbit data for the specified `target`,
870 rearrange it and return it as a NetCDF file. 887 rearrange it and return it as a NetCDF file.
871 `started_at` and `stopped_at` are expected to be UTC. 888 `started_at` and `stopped_at` are expected to be UTC.
@@ -883,6 +900,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at): @@ -883,6 +900,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
883 targets_configs.append(get_target_config(target)) 900 targets_configs.append(get_target_config(target))
884 if 0 == len(targets_configs): 901 if 0 == len(targets_configs):
885 abort(400, "No valid targets specified. What are you doing?") 902 abort(400, "No valid targets specified. What are you doing?")
  903 +
886 params = params.split(separator) 904 params = params.split(separator)
887 params.sort() 905 params.sort()
888 if 0 == len(params): 906 if 0 == len(params):
@@ -1009,12 +1027,6 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): @@ -1009,12 +1027,6 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1009 abort(400, "No valid targets specified. What are you doing?") 1027 abort(400, "No valid targets specified. What are you doing?")
1010 1028
1011 params = PARAMETERS.keys() 1029 params = PARAMETERS.keys()
1012 - # params = params.split(separator)  
1013 - # params.sort()  
1014 - # if 0 == len(params):  
1015 - # abort(400, "No valid parameters specified. What are you doing?")  
1016 - # if not is_list_in_list(params, PARAMETERS.keys()):  
1017 - # abort(400, "Some parameters are not recognized in '%s'." % str(params))  
1018 1030
1019 try: 1031 try:
1020 started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) 1032 started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
@@ -1036,21 +1048,18 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): @@ -1036,21 +1048,18 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1036 1048
1037 if not isfile(cdf_path): 1049 if not isfile(cdf_path):
1038 log.debug("Creating the CDF file '%s'..." % cdf_filename) 1050 log.debug("Creating the CDF file '%s'..." % cdf_filename)
1039 - environ['SPACEPY'] = CACHE_DIR  
1040 - environ['CDF_LIB'] = CDF_LIB  
1041 - try:  
1042 - # Importing pydcf will fail if the toolbox is not up to date.  
1043 - # todo: maybe move this to the daily cache warmup ?  
1044 - log.info("Updating spacepy's toolbox…")  
1045 - import spacepy.toolbox  
1046 - spacepy.toolbox.update()  
1047 - except Exception as e:  
1048 - log.error("Failed to update spacepy : %s." % e)  
1049 try: 1051 try:
1050 from spacepy import pycdf 1052 from spacepy import pycdf
1051 - except ImportError as e:  
1052 - log.error("Failed to import pycdf from spacepy : %s" % e)  
1053 - raise 1053 + except ImportError:
  1054 + # If spacepy's toolbox is not up-to-date, importing will fail.
  1055 + # So, let's update and try again !
  1056 + update_spacepy()
  1057 + try:
  1058 + from spacepy import pycdf
  1059 + except ImportError as e:
  1060 + log.error("Failed to import pycdf from spacepy : %s" % e)
  1061 + raise
  1062 +
1054 try: 1063 try:
1055 cdf_handle = pycdf.CDF(cdf_path, masterpath='') 1064 cdf_handle = pycdf.CDF(cdf_path, masterpath='')
1056 description = "Model and orbit data for %s." % \ 1065 description = "Model and orbit data for %s." % \
@@ -1074,15 +1083,26 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): @@ -1074,15 +1083,26 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1074 date = datetime.datetime.strptime(time_str, FILE_DATE_FMT) 1083 date = datetime.datetime.strptime(time_str, FILE_DATE_FMT)
1075 values.append(date) 1084 values.append(date)
1076 kt = "%s_time" % target_slug 1085 kt = "%s_time" % target_slug
  1086 + cdf_handle.new(kt, type=pycdf.const.CDF_EPOCH)
1077 cdf_handle[kt] = values 1087 cdf_handle[kt] = values
1078 - cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D" 1088 + # cdf_handle[kt].attrs['FIELDNAM'] = "Time since 0 A.D"
1079 1089
1080 for param in params: 1090 for param in params:
1081 k = "%s_%s" % (target_slug, param) 1091 k = "%s_%s" % (target_slug, param)
  1092 + # print("PARAM %s" % k)
1082 values = [] 1093 values = []
1083 i = available_params.index(param) 1094 i = available_params.index(param)
  1095 + has_nones = False
1084 for dkey in dkeys: 1096 for dkey in dkeys:
1085 - values.append(data[dkey][i]) 1097 + value = data[dkey][i]
  1098 + if value is None:
  1099 + has_nones = True
  1100 + values.append(value)
  1101 + if has_nones:
  1102 + # PyCDF hates it when there are Nones.
  1103 + # Since we don't know what value to set instead,
  1104 + # let's skip the param altogether.
  1105 + continue
1086 cdf_handle[k] = values 1106 cdf_handle[k] = values
1087 attrs = cdf_handle[k].attrs 1107 attrs = cdf_handle[k].attrs
1088 attrs['UNITS'] = PARAMETERS[param]['units'] 1108 attrs['UNITS'] = PARAMETERS[param]['units']
@@ -1102,11 +1122,14 @@ def download_targets_cdf(targets, inp, started_at, stopped_at): @@ -1102,11 +1122,14 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1102 value_xhee = data[dkey][index_x] 1122 value_xhee = data[dkey][index_x]
1103 value_yhee = data[dkey][index_y] 1123 value_yhee = data[dkey][index_y]
1104 # We've got some `None`s cropping up in the data sometimes. 1124 # We've got some `None`s cropping up in the data sometimes.
  1125 + # PyCDF does not digest Nones at all.
1105 # While they solve this upstream, let's make an ugly fix! 1126 # While they solve this upstream, let's make an ugly fix!
1106 if (value_xhee is not None) and (value_yhee is not None): 1127 if (value_xhee is not None) and (value_yhee is not None):
1107 values_xhee.append(value_xhee) 1128 values_xhee.append(value_xhee)
1108 values_yhee.append(value_yhee) 1129 values_yhee.append(value_yhee)
1109 else: 1130 else:
  1131 + values_xhee.append(0)
  1132 + values_yhee.append(0)
1110 log.warn("Orbit data for %s has NaNs." % target_slug) 1133 log.warn("Orbit data for %s has NaNs." % target_slug)
1111 cdf_handle[kx] = values_xhee 1134 cdf_handle[kx] = values_xhee
1112 cdf_handle[ky] = values_yhee 1135 cdf_handle[ky] = values_yhee
@@ -1174,10 +1197,12 @@ def cache_warmup(): @@ -1174,10 +1197,12 @@ def cache_warmup():
1174 inp = 'l1' # default input, maybe warm them all up ? 1197 inp = 'l1' # default input, maybe warm them all up ?
1175 1198
1176 targets = get_active_targets() 1199 targets = get_active_targets()
1177 - for target in targets:  
1178 - download_target_csv(target['slug'], inp, sta, sto)  
1179 targets_slugs = [target['slug'] for target in targets] 1200 targets_slugs = [target['slug'] for target in targets]
1180 targets_slugs.sort() 1201 targets_slugs.sort()
  1202 +
  1203 + update_spacepy()
  1204 + for target in targets:
  1205 + download_target_csv(target['slug'], inp, sta, sto)
1181 download_targets_cdf('-'.join(targets_slugs), inp, sta, sto) 1206 download_targets_cdf('-'.join(targets_slugs), inp, sta, sto)
1182 1207
1183 warmup_ended_at = datetime.datetime.now() 1208 warmup_ended_at = datetime.datetime.now()