CDPP / HELIOPROPA

Commit 180d7d976f197b5ff1da7df82e9aa9cbbe11c268

Authored by Goutte
1 parent 5f63c83b
Exists in master and in 2 other branches DEV, rhitier-dev

Refactor heavily.

Showing 1 changed file with 66 additions and 41 deletions   Show diff stats
Inline Side-by-side
web/run.py
  Diff comments   View file @180d7d9
... ... @@ -175,14 +175,18 @@ def datetime_from_list(time_list):
175 175 )
176 176  
177 177  
178   -def get_source_config(slug):
  178 +def get_target_config(slug):
179 179 for s in config['targets']: # dumb
180 180 if s['slug'] == slug:
181 181 return s
182   - raise Exception("No source found for slug '%s'." % slug)
  182 + raise Exception("No target found in configuration for '%s'." % slug)
183 183  
184 184  
185   -def retrieve_data(orbiter, what, started_at, stopped_at):
  185 +def check_target_config(slug):
  186 + get_target_config(slug)
  187 +
  188 +
  189 +def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
186 190 """
187 191 Handles remote querying Myriam's API, downloading, extracting and caching
188 192 the netCDF files.
... ... @@ -268,50 +272,59 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
268 272 return local_netc_files
269 273  
270 274  
271   -def generate_csv_contents(source_config, started_at, stopped_at):
  275 +# The available parameters in the generated CSV and NetCDF files.
  276 +# The order matters. If you change this you also need to change the
  277 +# innermost loop of `get_data_for_target`.
  278 +# The javascript knows the targets' properties under these names.
  279 +PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens',
  280 + 'angl', 'xhci', 'yhci')
  281 +
  282 +
  283 +def get_data_for_target(target_config, started_at, stopped_at):
  284 + """
  285 + :return: dict whose keys are datetime as str, values tuples of data
  286 + """
  287 + log.debug("Grabbing data for '%s'..." % target_config['slug'])
272 288 # @todo iterate on models when there are many
273 289 try:
274   - model_slug = source_config['models'][0]['slug']
  290 + model_slug = target_config['models'][0]['slug']
275 291 except Exception as e:
276 292 abort(500, "Invalid model configuration for '%s' : %s"
277   - % (source_config['slug'], str(e)))
  293 + % (target_config['slug'], str(e)))
  294 + try:
  295 + orbit_slug = target_config['orbit']['model']
  296 + except Exception as e:
  297 + abort(500, "Invalid orbit configuration for '%s' : %s"
  298 + % (target_config['slug'], str(e)))
278 299  
279 300 # Grab the list of netCDF files from Myriam's API
280 301 # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
281 302 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
282   - log.info("Generating CSV for '%s'..." % source_config['slug'])
283   - model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
284   - orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
285   -
286   - log.debug("Crunching CSV contents for '%s'..." % source_config['name'])
287   - si = StringIO.StringIO()
288   - cw = csv_writer(si)
289   - cw.writerow(( # the order matters !
290   - 'time',
291   - 'vrad', 'vtan', 'vlen',
292   - 'magn', 'temp', 'pdyn', 'dens', 'angl',
293   - 'xhci', 'yhci'
294   - ))
  303 + model_files = retrieve_amda_netcdf(target_config['slug'], model_slug,
  304 + started_at, stopped_at)
  305 + orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug,
  306 + started_at, stopped_at)
295 307  
296 308 precision = "%Y-%m-%dT%H" # model and orbits times are equal-ish
297   - orbits_data = {} # keys are datetime as str, values arrays of XY
298   - for orbits_file in orbits_files:
  309 + orbit_data = {} # keys are datetime as str, values arrays of XY
  310 + for orbit_file in orbit_files:
299 311 log.debug("%s: opening orbit NETCDF4 '%s'..." %
300   - (source_config['name'], orbits_file))
301   - cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
  312 + (target_config['name'], orbit_file))
  313 + cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
302 314 times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms
303 315 data_hci = cdf_handle.variables['HCI']
304 316 for time, datum_hci in zip(times, data_hci):
305 317 dtime = datetime_from_list(time)
306 318 if started_at <= dtime <= stopped_at:
307 319 dkey = dtime.strftime(precision)
308   - orbits_data[dkey] = datum_hci
  320 + orbit_data[dkey] = datum_hci
309 321 cdf_handle.close()
  322 +
310 323 all_data = {} # keys are datetime as str, values tuples of data
311 324 for model_file in model_files:
312 325 # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
313 326 log.debug("%s: opening model NETCDF4 '%s'..." %
314   - (source_config['name'], model_file))
  327 + (target_config['name'], model_file))
315 328 cdf_handle = Dataset(model_file, "r", format="NETCDF4")
316 329 times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms
317 330 data_v = cdf_handle.variables['V']
... ... @@ -321,7 +334,7 @@ def generate_csv_contents(source_config, started_at, stopped_at):
321 334 data_p = cdf_handle.variables['P_dyn']
322 335 data_d = cdf_handle.variables['Delta_angle']
323 336 log.debug("%s: aggregating data from '%s'..." %
324   - (source_config['name'], model_file))
  337 + (target_config['name'], model_file))
325 338 for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \
326 339 in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
327 340 vrad = datum_v[0]
... ... @@ -331,9 +344,9 @@ def generate_csv_contents(source_config, started_at, stopped_at):
331 344 dkey = dtime.strftime(precision)
332 345 x_hci = None
333 346 y_hci = None
334   - if dkey in orbits_data:
335   - x_hci = orbits_data[dkey][0]
336   - y_hci = orbits_data[dkey][1]
  347 + if dkey in orbit_data:
  348 + x_hci = orbit_data[dkey][0]
  349 + y_hci = orbit_data[dkey][1]
337 350 all_data[dkey] = (
338 351 dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
339 352 vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
... ... @@ -342,16 +355,28 @@ def generate_csv_contents(source_config, started_at, stopped_at):
342 355 )
343 356 cdf_handle.close()
344 357  
345   - log.debug("Writing and sorting CSV for '%s'..." % source_config['slug'])
  358 + return all_data
  359 +
  360 +
  361 +def generate_csv_contents(target_slug, started_at, stopped_at):
  362 + target_config = get_target_config(target_slug)
  363 + log.debug("Crunching CSV contents for '%s'..." % target_config['name'])
  364 + si = StringIO.StringIO()
  365 + cw = csv_writer(si)
  366 + cw.writerow(PROPERTIES)
  367 +
  368 + all_data = get_data_for_target(target_config, started_at, stopped_at)
  369 +
  370 + log.debug("Writing and sorting CSV for '%s'..." % target_config['slug'])
346 371 for dkey in sorted(all_data):
347 372 cw.writerow(all_data[dkey])
348 373  
349   - log.info("Generated CSV contents for '%s'." % source_config['slug'])
  374 + log.info("Generated CSV contents for '%s'." % target_config['slug'])
350 375 return si.getvalue()
351 376  
352 377  
353   -def generate_csv_file_if_needed(target_config, started_at, stopped_at):
354   - filename = "%s_%s_%s.csv" % (target_config['slug'],
  378 +def generate_csv_file_if_needed(target_slug, started_at, stopped_at):
  379 + filename = "%s_%s_%s.csv" % (target_slug,
355 380 started_at.strftime(FILE_DATE_FMT),
356 381 stopped_at.strftime(FILE_DATE_FMT))
357 382 local_csv_file = get_path("../cache/%s" % filename)
... ... @@ -359,7 +384,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at):
359 384 log.info("Generating CSV '%s'..." % local_csv_file)
360 385 try:
361 386 with open(local_csv_file, mode="w+") as f:
362   - f.write(generate_csv_contents(target_config,
  387 + f.write(generate_csv_contents(target_slug,
363 388 started_at=started_at,
364 389 stopped_at=stopped_at))
365 390 log.info("Generation of '%s' done." % filename)
... ... @@ -453,14 +478,14 @@ def home():
453 478 })
454 479  
455 480  
456   -@app.route("/<source>_<started_at>_<stopped_at>.csv")
457   -def get_target_csv(source, started_at, stopped_at):
  481 +@app.route("/<target>_<started_at>_<stopped_at>.csv")
  482 +def download_target_csv(target, started_at, stopped_at):
458 483 """
459 484 Grab data and orbit data for the specified `target`,
460 485 rearrange it and return it as a CSV file.
461 486 `started_at` and `stopped_at` should be UTC.
462 487 """
463   - source_config = get_source_config(source)
  488 + check_target_config(target)
464 489 try:
465 490 started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
466 491 except:
... ... @@ -470,11 +495,11 @@ def get_target_csv(source, started_at, stopped_at):
470 495 except:
471 496 abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
472 497  
473   - filename = "%s_%s_%s.csv" % (source,
  498 + filename = "%s_%s_%s.csv" % (target,
474 499 started_at.strftime(FILE_DATE_FMT),
475 500 stopped_at.strftime(FILE_DATE_FMT))
476 501 local_csv_file = get_path("../cache/%s" % filename)
477   - generate_csv_file_if_needed(source_config, started_at, stopped_at)
  502 + generate_csv_file_if_needed(target, started_at, stopped_at)
478 503 if not isfile(local_csv_file):
479 504 abort(500, "Could not cache CSV file at '%s'." % local_csv_file)
480 505  
... ... @@ -498,7 +523,7 @@ def download_targets_tarball(targets, started_at, stopped_at):
498 523 for target in targets:
499 524 if not target:
500 525 abort(400, "Invalid targets format : `%s`." % targets)
501   - targets_configs.append(get_source_config(target))
  526 + targets_configs.append(get_target_config(target))
502 527 if 0 == len(targets_configs):
503 528 abort(400, "No valid targets specified. What are you doing?")
504 529  
... ... @@ -524,7 +549,7 @@ def download_targets_tarball(targets, started_at, stopped_at):
524 549 local_csv_file = get_path("../cache/%s" % filename)
525 550 if not isfile(local_csv_file):
526 551 with open(local_csv_file, mode="w+") as f:
527   - f.write(generate_csv_contents(target_config,
  552 + f.write(generate_csv_contents(target_config['slug'],
528 553 started_at=started_at,
529 554 stopped_at=stopped_at))
530 555  
... ...