From 58bfe2813b911f5523ba5fdfa4316eb2f7280501 Mon Sep 17 00:00:00 2001 From: Goutte Date: Tue, 26 Sep 2017 14:29:20 +0200 Subject: [PATCH] Handle start and stop dates for the models as well. --- web/run.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------------- 1 file changed, 55 insertions(+), 70 deletions(-) diff --git a/web/run.py b/web/run.py index 8f00d72..93f0e0a 100755 --- a/web/run.py +++ b/web/run.py @@ -427,44 +427,24 @@ def get_data_for_target(target_config, started_at, stopped_at): abort(500, "Invalid orbit configuration for '%s' : %s" % (target_config['slug'], str(e))) - # try: - # started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) - # except: - # abort(400, "Invalid started_at config : '%s'." % started_at) - # try: - # stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT) - # except: - # abort(400, "Invalid stopped_at config : '%s'." % stopped_at) - - # Grab the list of netCDF files from Myriam's API - model_files = [] - for model in models: - model_files = model_files + retrieve_amda_netcdf( - target_config['slug'], model['slug'], started_at, stopped_at - ) - # Remove possible duplicates - model_files = set(model_files) - - # Let's return an empty dict when there's no data instead of crashing - # if not len(model_files): - # abort(500, "No model files found for '%s'." % target_config['slug']) - # if not len(orbit_files): - # abort(500, "No orbit files found for '%s'." % target_config['slug']) + def _sta_sto(_cnf, _sta, _sto): + if 'started_at' in _cnf: + _s0 = datetime.datetime.strptime(_cnf['started_at'], FILE_DATE_FMT) + _s0 = max(_s0, _sta) + else: + _s0 = _sta + if 'stopped_at' in _cnf: + _s1 = datetime.datetime.strptime(_cnf['stopped_at'], FILE_DATE_FMT) + _s1 = min(_s1, _sto) + else: + _s1 = _sto + return _s1, _s0 precision = "%Y-%m-%dT%H" # model and orbits times are only equal-ish orbit_data = {} # keys are datetime as str, values arrays of XY for orbit in orbits: - if 'started_at' in orbit: - s0 = datetime.datetime.strptime(orbit['started_at'], FILE_DATE_FMT) - s0 = max(s0, started_at) - else: - s0 = started_at - if 'stopped_at' in orbit: - s1 = datetime.datetime.strptime(orbit['stopped_at'], FILE_DATE_FMT) - s1 = min(s1, stopped_at) - else: - s1 = stopped_at + s0, s1 = _sta_sto(orbit, started_at, stopped_at) orbit_files = retrieve_amda_netcdf( target_config['slug'], orbit['slug'], s0, s1 @@ -493,43 +473,48 @@ def get_data_for_target(target_config, started_at, stopped_at): cdf_handle.close() all_data = {} # keys are datetime as str, values tuples of data - for model_file in model_files: - # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn - log.debug("%s: opening model NETCDF4 '%s'..." % - (target_config['name'], model_file)) - cdf_handle = Dataset(model_file, "r", format="NETCDF4") - times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms - data_v = cdf_handle.variables['V'] - data_b = cdf_handle.variables['B'] - data_t = cdf_handle.variables['T'] - data_n = cdf_handle.variables['N'] - data_p = cdf_handle.variables['P_dyn'] - data_d = cdf_handle.variables['Delta_angle'] - log.debug("%s: aggregating data from '%s'..." % - (target_config['name'], model_file)) - for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ - in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): - vrad = datum_v[0] - vtan = datum_v[1] - try: - dtime = datetime_from_list(time) - except Exception as e: - log.error("Failed to parse time from %s." % time) - raise e - if started_at <= dtime <= stopped_at: - dkey = round_time(dtime, 60*60).strftime(precision) - x_hee = None - y_hee = None - if dkey in orbit_data: - x_hee = orbit_data[dkey][0] - y_hee = orbit_data[dkey][1] - all_data[dkey] = ( - dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), - vrad, vtan, sqrt(vrad * vrad + vtan * vtan), - datum_b, datum_t, datum_n, datum_p, datum_d, - x_hee, y_hee - ) - cdf_handle.close() + for model in models: + s0, s1 = _sta_sto(model, started_at, stopped_at) + model_files = retrieve_amda_netcdf( + target_config['slug'], model['slug'], s0, s1 + ) + for model_file in model_files: + # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn + log.debug("%s: opening model NETCDF4 '%s'..." % + (target_config['name'], model_file)) + cdf_handle = Dataset(model_file, "r", format="NETCDF4") + times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms + data_v = cdf_handle.variables['V'] + data_b = cdf_handle.variables['B'] + data_t = cdf_handle.variables['T'] + data_n = cdf_handle.variables['N'] + data_p = cdf_handle.variables['P_dyn'] + data_d = cdf_handle.variables['Delta_angle'] + log.debug("%s: aggregating data from '%s'..." % + (target_config['name'], model_file)) + for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ + in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): + vrad = datum_v[0] + vtan = datum_v[1] + try: + dtime = datetime_from_list(time) + except Exception as e: + log.error("Failed to parse time from %s." % time) + raise e + if s0 <= dtime <= s1: + dkey = round_time(dtime, 60*60).strftime(precision) + x_hee = None + y_hee = None + if dkey in orbit_data: + x_hee = orbit_data[dkey][0] + y_hee = orbit_data[dkey][1] + all_data[dkey] = ( + dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), + vrad, vtan, sqrt(vrad * vrad + vtan * vtan), + datum_b, datum_t, datum_n, datum_p, datum_d, + x_hee, y_hee + ) + cdf_handle.close() return all_data -- libgit2 0.21.2