diff --git a/web/run.py b/web/run.py index 2a17f41..9cc15a7 100755 --- a/web/run.py +++ b/web/run.py @@ -175,14 +175,18 @@ def datetime_from_list(time_list): ) -def get_source_config(slug): +def get_target_config(slug): for s in config['targets']: # dumb if s['slug'] == slug: return s - raise Exception("No source found for slug '%s'." % slug) + raise Exception("No target found in configuration for '%s'." % slug) -def retrieve_data(orbiter, what, started_at, stopped_at): +def check_target_config(slug): + get_target_config(slug) + + +def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): """ Handles remote querying Myriam's API, downloading, extracting and caching the netCDF files. @@ -268,50 +272,59 @@ def retrieve_data(orbiter, what, started_at, stopped_at): return local_netc_files -def generate_csv_contents(source_config, started_at, stopped_at): +# The available parameters in the generated CSV and NetCDF files. +# The order matters. If you change this you also need to change the +# innermost loop of `get_data_for_target`. +# The javascript knows the targets' properties under these names. +PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens', + 'angl', 'xhci', 'yhci') + + +def get_data_for_target(target_config, started_at, stopped_at): + """ + :return: dict whose keys are datetime as str, values tuples of data + """ + log.debug("Grabbing data for '%s'..." % target_config['slug']) # @todo iterate on models when there are many try: - model_slug = source_config['models'][0]['slug'] + model_slug = target_config['models'][0]['slug'] except Exception as e: abort(500, "Invalid model configuration for '%s' : %s" - % (source_config['slug'], str(e))) + % (target_config['slug'], str(e))) + try: + orbit_slug = target_config['orbit']['model'] + except Exception as e: + abort(500, "Invalid orbit configuration for '%s' : %s" + % (target_config['slug'], str(e))) # Grab the list of netCDF files from Myriam's API # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz - log.info("Generating CSV for '%s'..." % source_config['slug']) - model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at) - orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at) - - log.debug("Crunching CSV contents for '%s'..." % source_config['name']) - si = StringIO.StringIO() - cw = csv_writer(si) - cw.writerow(( # the order matters ! - 'time', - 'vrad', 'vtan', 'vlen', - 'magn', 'temp', 'pdyn', 'dens', 'angl', - 'xhci', 'yhci' - )) + model_files = retrieve_amda_netcdf(target_config['slug'], model_slug, + started_at, stopped_at) + orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug, + started_at, stopped_at) precision = "%Y-%m-%dT%H" # model and orbits times are equal-ish - orbits_data = {} # keys are datetime as str, values arrays of XY - for orbits_file in orbits_files: + orbit_data = {} # keys are datetime as str, values arrays of XY + for orbit_file in orbit_files: log.debug("%s: opening orbit NETCDF4 '%s'..." % - (source_config['name'], orbits_file)) - cdf_handle = Dataset(orbits_file, "r", format="NETCDF4") + (target_config['name'], orbit_file)) + cdf_handle = Dataset(orbit_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_hci = cdf_handle.variables['HCI'] for time, datum_hci in zip(times, data_hci): dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: dkey = dtime.strftime(precision) - orbits_data[dkey] = datum_hci + orbit_data[dkey] = datum_hci cdf_handle.close() + all_data = {} # keys are datetime as str, values tuples of data for model_file in model_files: # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn log.debug("%s: opening model NETCDF4 '%s'..." % - (source_config['name'], model_file)) + (target_config['name'], model_file)) cdf_handle = Dataset(model_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_v = cdf_handle.variables['V'] @@ -321,7 +334,7 @@ def generate_csv_contents(source_config, started_at, stopped_at): data_p = cdf_handle.variables['P_dyn'] data_d = cdf_handle.variables['Delta_angle'] log.debug("%s: aggregating data from '%s'..." % - (source_config['name'], model_file)) + (target_config['name'], model_file)) for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): vrad = datum_v[0] @@ -331,9 +344,9 @@ def generate_csv_contents(source_config, started_at, stopped_at): dkey = dtime.strftime(precision) x_hci = None y_hci = None - if dkey in orbits_data: - x_hci = orbits_data[dkey][0] - y_hci = orbits_data[dkey][1] + if dkey in orbit_data: + x_hci = orbit_data[dkey][0] + y_hci = orbit_data[dkey][1] all_data[dkey] = ( dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), vrad, vtan, sqrt(vrad * vrad + vtan * vtan), @@ -342,16 +355,28 @@ def generate_csv_contents(source_config, started_at, stopped_at): ) cdf_handle.close() - log.debug("Writing and sorting CSV for '%s'..." % source_config['slug']) + return all_data + + +def generate_csv_contents(target_slug, started_at, stopped_at): + target_config = get_target_config(target_slug) + log.debug("Crunching CSV contents for '%s'..." % target_config['name']) + si = StringIO.StringIO() + cw = csv_writer(si) + cw.writerow(PROPERTIES) + + all_data = get_data_for_target(target_config, started_at, stopped_at) + + log.debug("Writing and sorting CSV for '%s'..." % target_config['slug']) for dkey in sorted(all_data): cw.writerow(all_data[dkey]) - log.info("Generated CSV contents for '%s'." % source_config['slug']) + log.info("Generated CSV contents for '%s'." % target_config['slug']) return si.getvalue() -def generate_csv_file_if_needed(target_config, started_at, stopped_at): - filename = "%s_%s_%s.csv" % (target_config['slug'], +def generate_csv_file_if_needed(target_slug, started_at, stopped_at): + filename = "%s_%s_%s.csv" % (target_slug, started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) @@ -359,7 +384,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at): log.info("Generating CSV '%s'..." % local_csv_file) try: with open(local_csv_file, mode="w+") as f: - f.write(generate_csv_contents(target_config, + f.write(generate_csv_contents(target_slug, started_at=started_at, stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) @@ -453,14 +478,14 @@ def home(): }) -@app.route("/__.csv") -def get_target_csv(source, started_at, stopped_at): +@app.route("/__.csv") +def download_target_csv(target, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a CSV file. `started_at` and `stopped_at` should be UTC. """ - source_config = get_source_config(source) + check_target_config(target) try: started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) except: @@ -470,11 +495,11 @@ def get_target_csv(source, started_at, stopped_at): except: abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) - filename = "%s_%s_%s.csv" % (source, + filename = "%s_%s_%s.csv" % (target, started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) - generate_csv_file_if_needed(source_config, started_at, stopped_at) + generate_csv_file_if_needed(target, started_at, stopped_at) if not isfile(local_csv_file): abort(500, "Could not cache CSV file at '%s'." % local_csv_file) @@ -498,7 +523,7 @@ def download_targets_tarball(targets, started_at, stopped_at): for target in targets: if not target: abort(400, "Invalid targets format : `%s`." % targets) - targets_configs.append(get_source_config(target)) + targets_configs.append(get_target_config(target)) if 0 == len(targets_configs): abort(400, "No valid targets specified. What are you doing?") @@ -524,7 +549,7 @@ def download_targets_tarball(targets, started_at, stopped_at): local_csv_file = get_path("../cache/%s" % filename) if not isfile(local_csv_file): with open(local_csv_file, mode="w+") as f: - f.write(generate_csv_contents(target_config, + f.write(generate_csv_contents(target_config['slug'], started_at=started_at, stopped_at=stopped_at)) -- libgit2 0.21.2