Refactor heavily.

Goutte
1 parent 5f63c83b
Showing 1 changed file with 66 additions and 41 deletions Show diff stats
web/run.py
@@ -175,14 +175,18 @@ def datetime_from_list(time_list):
     )
-def get_source_config(slug):
+def get_target_config(slug):
     for s in config['targets']:  # dumb
         if s['slug'] == slug:
             return s
-    raise Exception("No source found for slug '%s'." % slug)
+    raise Exception("No target found in configuration for '%s'." % slug)
-def retrieve_data(orbiter, what, started_at, stopped_at):
+def check_target_config(slug):
+    get_target_config(slug)
+
+
+def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
     """
     Handles remote querying Myriam's API, downloading, extracting and caching
     the netCDF files.
@@ -268,50 +272,59 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
     return local_netc_files
-def generate_csv_contents(source_config, started_at, stopped_at):
+# The available parameters in the generated CSV and NetCDF files.
+# The order matters. If you change this you also need to change the
+# innermost loop of `get_data_for_target`.
+# The javascript knows the targets' properties under these names.
+PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens',
+              'angl', 'xhci', 'yhci')
+
+
+def get_data_for_target(target_config, started_at, stopped_at):
+    """
+    :return: dict whose keys are datetime as str, values tuples of data
+    """
+    log.debug("Grabbing data for '%s'..." % target_config['slug'])
     # @todo iterate on models when there are many
     try:
-        model_slug = source_config['models'][0]['slug']
+        model_slug = target_config['models'][0]['slug']
     except Exception as e:
         abort(500, "Invalid model configuration for '%s' : %s"
-                   % (source_config['slug'], str(e)))
+              % (target_config['slug'], str(e)))
+    try:
+        orbit_slug = target_config['orbit']['model']
+    except Exception as e:
+        abort(500, "Invalid orbit configuration for '%s' : %s"
+              % (target_config['slug'], str(e)))
     # Grab the list of netCDF files from Myriam's API
     # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
     # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
-    log.info("Generating CSV for '%s'..." % source_config['slug'])
-    model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
-    orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
-
-    log.debug("Crunching CSV contents for '%s'..." % source_config['name'])
-    si = StringIO.StringIO()
-    cw = csv_writer(si)
-    cw.writerow((  # the order matters !
-        'time',
-        'vrad', 'vtan', 'vlen',
-        'magn', 'temp', 'pdyn', 'dens', 'angl',
-        'xhci', 'yhci'
-    ))
+    model_files = retrieve_amda_netcdf(target_config['slug'], model_slug,
+                                       started_at, stopped_at)
+    orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug,
+                                       started_at, stopped_at)
     precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
-    orbits_data = {}  # keys are datetime as str, values arrays of XY
-    for orbits_file in orbits_files:
+    orbit_data = {}  # keys are datetime as str, values arrays of XY
+    for orbit_file in orbit_files:
         log.debug("%s: opening orbit NETCDF4 '%s'..." %
-                  (source_config['name'], orbits_file))
-        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
+                  (target_config['name'], orbit_file))
+        cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
         times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
         data_hci = cdf_handle.variables['HCI']
         for time, datum_hci in zip(times, data_hci):
             dtime = datetime_from_list(time)
             if started_at <= dtime <= stopped_at:
                 dkey = dtime.strftime(precision)
-                orbits_data[dkey] = datum_hci
+                orbit_data[dkey] = datum_hci
         cdf_handle.close()
+
     all_data = {}  # keys are datetime as str, values tuples of data
     for model_file in model_files:
         # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
         log.debug("%s: opening model NETCDF4 '%s'..." %
-                  (source_config['name'], model_file))
+                  (target_config['name'], model_file))
         cdf_handle = Dataset(model_file, "r", format="NETCDF4")
         times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
         data_v = cdf_handle.variables['V']
@@ -321,7 +334,7 @@ def generate_csv_contents(source_config, started_at, stopped_at):
         data_p = cdf_handle.variables['P_dyn']
         data_d = cdf_handle.variables['Delta_angle']
         log.debug("%s: aggregating data from '%s'..." %
-                  (source_config['name'], model_file))
+                  (target_config['name'], model_file))
         for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \
                 in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
             vrad = datum_v[0]
@@ -331,9 +344,9 @@ def generate_csv_contents(source_config, started_at, stopped_at):
                 dkey = dtime.strftime(precision)
                 x_hci = None
                 y_hci = None
-                if dkey in orbits_data:
-                    x_hci = orbits_data[dkey][0]
-                    y_hci = orbits_data[dkey][1]
+                if dkey in orbit_data:
+                    x_hci = orbit_data[dkey][0]
+                    y_hci = orbit_data[dkey][1]
                 all_data[dkey] = (
                     dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                     vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
@@ -342,16 +355,28 @@ def generate_csv_contents(source_config, started_at, stopped_at):
                 )
         cdf_handle.close()
-    log.debug("Writing and sorting CSV for '%s'..." % source_config['slug'])
+    return all_data
+
+
+def generate_csv_contents(target_slug, started_at, stopped_at):
+    target_config = get_target_config(target_slug)
+    log.debug("Crunching CSV contents for '%s'..." % target_config['name'])
+    si = StringIO.StringIO()
+    cw = csv_writer(si)
+    cw.writerow(PROPERTIES)
+
+    all_data = get_data_for_target(target_config, started_at, stopped_at)
+
+    log.debug("Writing and sorting CSV for '%s'..." % target_config['slug'])
     for dkey in sorted(all_data):
         cw.writerow(all_data[dkey])
-    log.info("Generated CSV contents for '%s'." % source_config['slug'])
+    log.info("Generated CSV contents for '%s'." % target_config['slug'])
     return si.getvalue()
-def generate_csv_file_if_needed(target_config, started_at, stopped_at):
-    filename = "%s_%s_%s.csv" % (target_config['slug'],
+def generate_csv_file_if_needed(target_slug, started_at, stopped_at):
+    filename = "%s_%s_%s.csv" % (target_slug,
                                  started_at.strftime(FILE_DATE_FMT),
                                  stopped_at.strftime(FILE_DATE_FMT))
     local_csv_file = get_path("../cache/%s" % filename)
@@ -359,7 +384,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at):
         log.info("Generating CSV '%s'..." % local_csv_file)
         try:
             with open(local_csv_file, mode="w+") as f:
-                f.write(generate_csv_contents(target_config,
+                f.write(generate_csv_contents(target_slug,
                                               started_at=started_at,
                                               stopped_at=stopped_at))
             log.info("Generation of '%s' done." % filename)
@@ -453,14 +478,14 @@ def home():
     })
-@app.route("/<source>_<started_at>_<stopped_at>.csv")
-def get_target_csv(source, started_at, stopped_at):
+@app.route("/<target>_<started_at>_<stopped_at>.csv")
+def download_target_csv(target, started_at, stopped_at):
     """
     Grab data and orbit data for the specified `target`,
     rearrange it and return it as a CSV file.
     `started_at` and `stopped_at` should be UTC.
     """
-    source_config = get_source_config(source)
+    check_target_config(target)
     try:
         started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
     except:
@@ -470,11 +495,11 @@ def get_target_csv(source, started_at, stopped_at):
     except:
         abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
-    filename = "%s_%s_%s.csv" % (source,
+    filename = "%s_%s_%s.csv" % (target,
                                  started_at.strftime(FILE_DATE_FMT),
                                  stopped_at.strftime(FILE_DATE_FMT))
     local_csv_file = get_path("../cache/%s" % filename)
-    generate_csv_file_if_needed(source_config, started_at, stopped_at)
+    generate_csv_file_if_needed(target, started_at, stopped_at)
     if not isfile(local_csv_file):
         abort(500, "Could not cache CSV file at '%s'." % local_csv_file)
@@ -498,7 +523,7 @@ def download_targets_tarball(targets, started_at, stopped_at):
     for target in targets:
         if not target:
             abort(400, "Invalid targets format : `%s`." % targets)
-        targets_configs.append(get_source_config(target))
+        targets_configs.append(get_target_config(target))
     if 0 == len(targets_configs):
         abort(400, "No valid targets specified. What are you doing?")
@@ -524,7 +549,7 @@ def download_targets_tarball(targets, started_at, stopped_at):
             local_csv_file = get_path("../cache/%s" % filename)
             if not isfile(local_csv_file):
                 with open(local_csv_file, mode="w+") as f:
-                    f.write(generate_csv_contents(target_config,
+                    f.write(generate_csv_contents(target_config['slug'],
                                                   started_at=started_at,
                                                   stopped_at=stopped_at))