import StringIO import datetime import time import gzip import json import logging import random import tarfile import urllib from csv import writer as csv_writer from math import sqrt from os import environ, remove as removefile from os.path import isfile, join, abspath, dirname from flask import Flask from flask import request from flask import url_for, send_from_directory, abort as abort_flask from jinja2 import Environment, FileSystemLoader from netCDF4 import Dataset, date2num from yaml import load as yaml_load # PATH RELATIVITY ############################################################# THIS_DIRECTORY = dirname(abspath(__file__)) def get_path(relative_path): """Get an absolute path from the relative path to this script directory.""" return abspath(join(THIS_DIRECTORY, relative_path)) # COLLECT GLOBAL INFORMATION FROM SOURCES ##################################### # VERSION with open(get_path('../VERSION'), 'r') as version_file: version = version_file.read().strip() # CONFIG with open(get_path('../config.yml'), 'r') as config_file: config = yaml_load(config_file.read()) FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S" # LOGGING ##################################################################### log = logging.getLogger("HelioPropa") log.setLevel(logging.DEBUG) # log.setLevel(logging.ERROR) # <-- set log level here ! logHandler = logging.FileHandler(get_path('run.log')) logHandler.setFormatter(logging.Formatter( "%(asctime)s - %(levelname)s - %(message)s" )) log.addHandler(logHandler) # SETUP FLASK ENGINE ########################################################## app = Flask(__name__, root_path=THIS_DIRECTORY) app.debug = environ.get('DEBUG') == 'true' if app.debug: log.info("Starting Flask app IN DEBUG MODE...") else: log.info("Starting Flask app...") # SETUP JINJA2 TEMPLATE ENGINE ################################################ def static_global(filename): return url_for('static', filename=filename) def shuffle_filter(seq): """ This shuffles the sequence it is applied to. Jinja2 _should_ provide this. """ try: result = list(seq) random.shuffle(result) return result except: return seq def markdown_filter(value, nl2br=False, p=True): """ Converts markdown into html. nl2br: set to True to replace line breaks with
tags p: set to False to remove the enclosing

tags """ from markdown import markdown from markdown.extensions.nl2br import Nl2BrExtension from markdown.extensions.abbr import AbbrExtension extensions = [AbbrExtension()] if nl2br is True: extensions.append(Nl2BrExtension()) markdowned = markdown(value, output_format='html5', extensions=extensions) if p is False: markdowned = markdowned.replace(r"

", "").replace(r"

", "") return markdowned tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]), trim_blocks=True, lstrip_blocks=True) tpl_engine.globals.update( url_for=url_for, static=static_global, ) tpl_engine.filters['markdown'] = markdown_filter tpl_engine.filters['md'] = markdown_filter tpl_engine.filters['shuffle'] = shuffle_filter tpl_global_vars = { 'request': request, 'version': version, 'config': config, 'now': datetime.datetime.now(), } # HELPERS ##################################################################### def abort(code, message): log.error(message) abort_flask(code, message) def render_view(view, context=None): """ A simple helper to render [view] template with [context] vars. It automatically adds the global template vars defined above, too. It returns a string, usually the HTML contents to display. """ context = {} if context is None else context return tpl_engine.get_template(view).render( dict(tpl_global_vars.items() + context.items()) ) # def render_page(page, title="My Page", context=None): # """ # A simple helper to render the md_page.html template with [context] vars & # the additional contents of `page/[page].md` in the `md_page` variable. # It automagically adds the global template vars defined above, too. # It returns a string, usually the HTML contents to display. # """ # if context is None: # context = {} # context['title'] = title # context['md_page'] = '' # with file(get_path('page/%s.md' % page)) as f: # context['md_page'] = f.read() # return tpl_engine.get_template('md_page.html').render( # dict(tpl_global_vars.items() + context.items()) # ) def is_list_in_list(needle, haystack): for n in needle: if n not in haystack: return False return True def datetime_from_list(time_list): """ Datetimes in retrieved CDFs are stored as lists of numbers, with DayOfYear starting at 0. We want it starting at 1 for default parsers. """ # Day Of Year starts at 0, but for our datetime parser it starts at 1 doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1) return datetime.datetime.strptime( "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])), "%Y%j%H%M%S%f" ) def get_target_config(slug): for s in config['targets']: # dumb if s['slug'] == slug: return s raise Exception("No target found in configuration for '%s'." % slug) def check_target_config(slug): get_target_config(slug) def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): """ Handles remote querying Myriam's API, downloading, extracting and caching the netCDF files. :param orbiter: key of the source in the YAML config :param what: either 'model' or 'orbit', a key in the config of the source :param started_at: :param stopped_at: :return: a list of local file paths to netCDF (.nc) files """ url = config['amda'].format( dataSet=what, startTime=started_at.isoformat(), stopTime=stopped_at.isoformat() ) log.info("Fetching remote gzip files list at '%s'." % url) retries = 0 success = False errors = [] remote_gzip_files = [] while not success and retries < 3: try: response = urllib.urlopen(url) remote_gzip_files = json.loads(response.read()) if not remote_gzip_files: raise Exception("Failed to fetch data at '%s'." % url) if remote_gzip_files == 'NODATASET': raise Exception("API says there's no dataset at '%s'." % url) if remote_gzip_files == 'ERROR': raise Exception("API returned an error at '%s'." % url) if remote_gzip_files == ['OUTOFTIME']: # it happens raise Exception("API says it's out of time at '%s'." % url) success = True except Exception as e: log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message)) remote_gzip_files = [] errors.append(e) finally: retries += 1 if not remote_gzip_files: abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" % (orbiter, url, errors)) else: remote_gzip_files = list(set(remote_gzip_files)) log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files)) local_gzip_files = [] for remote_gzip_file in remote_gzip_files: # hotfixes to remove when fixed upstream @Myriam if remote_gzip_file in ['OUTOFTIME', 'ERROR']: continue # sometimes half the response is okay, the other not if remote_gzip_file.endswith('/.gz'): continue remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1) ################################################ filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1]) local_gzip_file = get_path("../cache/%s" % filename) local_gzip_files.append(local_gzip_file) if not isfile(local_gzip_file): log.debug("Retrieving '%s'..." % local_gzip_file) urllib.urlretrieve(remote_gzip_file, local_gzip_file) log.debug("Retrieved '%s'." % local_gzip_file) local_netc_files = [] for local_gzip_file in local_gzip_files: local_netc_file = local_gzip_file[0:-3] local_netc_files.append(local_netc_file) log.debug("Unzipping '%s'..." % local_gzip_file) success = True try: with gzip.open(local_gzip_file, 'rb') as f: file_content = f.read() with open(local_netc_file, 'w+b') as g: g.write(file_content) except Exception as e: success = False log.warning("Cannot process gz file '%s' from '%s' : %s" % (local_gzip_file, url, e)) if success: log.debug("Unzipped '%s'." % local_gzip_file) return local_netc_files # The available parameters in the generated CSV and NetCDF files. # The order matters. If you change this you also need to change the # innermost loop of `get_data_for_target`. # The javascript knows the targets' properties under these names. # DEPRECATED, use PARAMETERS instead PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens', 'angl', 'xhci', 'yhci') PARAMETERS = { 'pdyn': { 'slug': 'pdyn', 'name': 'Dyn. Pressure', 'unit': 'nPa', }, 'vlen': { 'slug': 'vlen', 'name': 'Velocity', 'unit': 'km/s', }, 'magn': { 'slug': 'magn', 'name': 'Magnetism', 'unit': 'nT', }, 'temp': { 'slug': 'temp', 'name': 'Temperature', 'unit': 'K', }, 'dens': { 'slug': 'dens', 'name': 'Density', 'unit': 'cm^-3', }, 'angl': { 'slug': 'angl', 'name': 'Angle T-S-E', 'unit': 'deg', }, } def get_data_for_target(target_config, started_at, stopped_at): """ :return: dict whose keys are datetime as str, values tuples of data """ log.debug("Grabbing data for '%s'..." % target_config['slug']) # @todo iterate on models when there are many try: model_slug = target_config['models'][0]['slug'] except Exception as e: abort(500, "Invalid model configuration for '%s' : %s" % (target_config['slug'], str(e))) try: orbit_slug = target_config['orbit']['model'] except Exception as e: abort(500, "Invalid orbit configuration for '%s' : %s" % (target_config['slug'], str(e))) # Grab the list of netCDF files from Myriam's API # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz model_files = retrieve_amda_netcdf(target_config['slug'], model_slug, started_at, stopped_at) orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug, started_at, stopped_at) precision = "%Y-%m-%dT%H" # model and orbits times are equal-ish orbit_data = {} # keys are datetime as str, values arrays of XY for orbit_file in orbit_files: log.debug("%s: opening orbit NETCDF4 '%s'..." % (target_config['name'], orbit_file)) cdf_handle = Dataset(orbit_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_hci = cdf_handle.variables['HCI'] for time, datum_hci in zip(times, data_hci): dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: dkey = dtime.strftime(precision) orbit_data[dkey] = datum_hci cdf_handle.close() all_data = {} # keys are datetime as str, values tuples of data for model_file in model_files: # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn log.debug("%s: opening model NETCDF4 '%s'..." % (target_config['name'], model_file)) cdf_handle = Dataset(model_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_v = cdf_handle.variables['V'] data_b = cdf_handle.variables['B'] data_t = cdf_handle.variables['T'] data_n = cdf_handle.variables['N'] data_p = cdf_handle.variables['P_dyn'] data_d = cdf_handle.variables['Delta_angle'] log.debug("%s: aggregating data from '%s'..." % (target_config['name'], model_file)) for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): vrad = datum_v[0] vtan = datum_v[1] dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: dkey = dtime.strftime(precision) x_hci = None y_hci = None if dkey in orbit_data: x_hci = orbit_data[dkey][0] y_hci = orbit_data[dkey][1] all_data[dkey] = ( dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), vrad, vtan, sqrt(vrad * vrad + vtan * vtan), datum_b, datum_t, datum_n, datum_p, datum_d, x_hci, y_hci ) cdf_handle.close() return all_data def generate_csv_contents(target_slug, started_at, stopped_at): target_config = get_target_config(target_slug) log.debug("Crunching CSV contents for '%s'..." % target_config['name']) si = StringIO.StringIO() cw = csv_writer(si) cw.writerow(PROPERTIES) all_data = get_data_for_target(target_config, started_at, stopped_at) log.debug("Writing and sorting CSV for '%s'..." % target_config['slug']) for dkey in sorted(all_data): cw.writerow(all_data[dkey]) log.info("Generated CSV contents for '%s'." % target_config['slug']) return si.getvalue() def generate_csv_file_if_needed(target_slug, started_at, stopped_at): filename = "%s_%s_%s.csv" % (target_slug, started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) if not isfile(local_csv_file): log.info("Generating CSV '%s'..." % local_csv_file) try: with open(local_csv_file, mode="w+") as f: f.write(generate_csv_contents(target_slug, started_at=started_at, stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) except Exception as e: if isfile(local_csv_file): log.warn("Removing failed CSV '%s'..." % local_csv_file) removefile(local_csv_file) abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) def remove_files_created_before(date, in_directory): """ Will throw if something horrible happens. Does not remove recursively (could be done with os.walk if needed). Does not remove directories either. :param date: datetime object :param in_directory: absolute path to directory :return: """ import os import time secs = time.mktime(date.timetuple()) if not os.path.isdir(in_directory): raise ValueError("No directory to clean at '%s'.") removed_files = [] for file_name in os.listdir(in_directory): file_path = os.path.join(in_directory, file_name) if os.path.isfile(file_path): t = os.stat(file_path) if t.st_ctime < secs: os.remove(file_path) removed_files.append(file_path) return removed_files def get_hit_counter(): hit_count_path = get_path("../VISITS") if isfile(hit_count_path): hit_count = int(open(hit_count_path).read()) else: hit_count = 1 return hit_count def increment_hit_counter(): hit_count_path = get_path("../VISITS") if isfile(hit_count_path): hit_count = int(open(hit_count_path).read()) hit_count += 1 else: hit_count = 1 hit_counter_file = open(hit_count_path, 'w') hit_counter_file.write(str(hit_count)) hit_counter_file.close() return hit_count tpl_global_vars['visits'] = get_hit_counter() # ROUTING ##################################################################### @app.route('/favicon.ico') def favicon(): return send_from_directory( join(app.root_path, 'static', 'img'), 'favicon.ico', mimetype='image/vnd.microsoft.icon' ) @app.route("/") @app.route("/home.html") @app.route("/index.html") def home(): increment_hit_counter() return render_view('home.html.jinja2', { 'targets': config['targets'], 'planets': [s for s in config['targets'] if s['type'] == 'planet'], 'probes': [s for s in config['targets'] if s['type'] == 'probe'], 'comets': [s for s in config['targets'] if s['type'] == 'comet'], 'visits': get_hit_counter(), }) @app.route("/__.csv") def download_target_csv(target, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a CSV file. `started_at` and `stopped_at` should be UTC. """ check_target_config(target) try: started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT) except: abort(400, "Invalid started_at parameter : '%s'." % started_at) try: stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT) except: abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) filename = "%s_%s_%s.csv" % (target, started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) generate_csv_file_if_needed(target, started_at, stopped_at) if not isfile(local_csv_file): abort(500, "Could not cache CSV file at '%s'." % local_csv_file) return send_from_directory(get_path("../cache/"), filename) @app.route("/__.tar.gz") def download_targets_tarball(targets, started_at, stopped_at): """ Grab data and orbit data for each of the specified `targets`, in their own CSV file, and make a tarball of them. `started_at` and `stopped_at` should be UTC strings. targets: string list of targets' slugs, separated by `-`. """ separator = '-' targets = targets.split(separator) targets.sort() targets_configs = [] for target in targets: if not target: abort(400, "Invalid targets format : `%s`." % targets) targets_configs.append(get_target_config(target)) if 0 == len(targets_configs): abort(400, "No valid targets specified. What are you doing?") date_fmt = FILE_DATE_FMT try: started_at = datetime.datetime.strptime(started_at, date_fmt) except: abort(400, "Invalid started_at parameter : '%s'." % started_at) try: stopped_at = datetime.datetime.strptime(stopped_at, date_fmt) except: abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) sta = started_at.strftime(date_fmt) sto = stopped_at.strftime(date_fmt) gzip_filename = "%s_%s_%s.tar.gz" % (separator.join(targets), sta, sto) local_gzip_file = get_path("../cache/%s" % gzip_filename) if not isfile(local_gzip_file): log.debug("Creating the CSV files for the tarball...") for target_config in targets_configs: filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto) local_csv_file = get_path("../cache/%s" % filename) if not isfile(local_csv_file): with open(local_csv_file, mode="w+") as f: f.write(generate_csv_contents(target_config['slug'], started_at=started_at, stopped_at=stopped_at)) log.debug("Creating the tarball '%s'..." % local_gzip_file) with tarfile.open(local_gzip_file, "w:gz") as tar: for target_config in targets_configs: filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto) local_csv_file = get_path("../cache/%s" % filename) tar.add(local_csv_file, arcname=filename) if not isfile(local_gzip_file): abort(500, "No tarball to serve. Looked at '%s'." % local_gzip_file) return send_from_directory(get_path("../cache/"), gzip_filename) @app.route("/___.nc") def download_targets_netcdf(targets, params, started_at, stopped_at): """ Grab data and orbit data for the specified `target`, rearrange it and return it as a CSV file. `started_at` and `stopped_at` should be UTC. targets: string list of targets' slugs, separated by `-`. params: string list of targets' parameters, separated by `-`. """ separator = '-' targets = targets.split(separator) targets.sort() targets_configs = [] for target in targets: if not target: abort(400, "Invalid targets format : `%s`." % targets) targets_configs.append(get_target_config(target)) if 0 == len(targets_configs): abort(400, "No valid targets specified. What are you doing?") params = params.split(separator) params.sort() if 0 == len(params): abort(400, "No valid parameters specified. What are you doing?") if not is_list_in_list(params, PARAMETERS.keys()): abort(400, "Some parameters are not recognized in '%s'." % str(params)) date_fmt = FILE_DATE_FMT try: started_at = datetime.datetime.strptime(started_at, date_fmt) except: abort(400, "Invalid started_at parameter : '%s'." % started_at) try: stopped_at = datetime.datetime.strptime(stopped_at, date_fmt) except: abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at) sta = started_at.strftime(date_fmt) sto = stopped_at.strftime(date_fmt) nc_filename = "%s_%s_%s_%s.nc" % \ (separator.join(targets), separator.join(params), sta, sto) nc_path = get_path("../cache/%s" % nc_filename) if not isfile(nc_path): log.debug("Creating the NetCDF file '%s'..." % nc_filename) nc_handle = Dataset(nc_path, "w", format="NETCDF4") try: nc_handle.description = "TODO" # todo nc_handle.history = "Created " + time.ctime(time.time()) nc_handle.source = "Transplanet (CDDP)" available_params = list(PROPERTIES) for target in targets_configs: target_slug = target['slug'] log.debug("Adding group '%s' to the NetCDF..." % target_slug) nc_group = nc_handle.createGroup(target_slug) data = get_data_for_target(target, started_at, stopped_at) dkeys = sorted(data) dimension = 'dim_'+target_slug nc_handle.createDimension(dimension, len(dkeys)) # TIME # nc_time = nc_group.createVariable('time', 'i8', (dimension,)) nc_time.units = "hours since 1970-01-01 00:00:00" nc_time.calendar = "standard" times = [] for dkey in dkeys: time_as_string = data[dkey][0][:-6] # remove +00:00 tail date = datetime.datetime.strptime(time_as_string, date_fmt) times.append(date2num( date, units=nc_time.units, calendar=nc_time.calendar )) nc_time[:] = times # SELECTED PARAMETERS # nc_vars = [] indices = [] for param in params: indices.append(available_params.index(param)) nc_var = nc_group.createVariable(param, 'f8', (dimension,)) nc_var.units = PARAMETERS[param]['unit'] nc_vars.append(nc_var) for i, nc_var in enumerate(nc_vars): index = indices[i] values = [] for dkey in dkeys: dval = data[dkey] values.append(dval[index]) nc_var[:] = values # ORBIT # nc_x = nc_group.createVariable('xhci', 'f8', (dimension,)) nc_x.units = 'Au' nc_y = nc_group.createVariable('yhci', 'f8', (dimension,)) nc_y.units = 'Au' values_x = [] values_y = [] index_x = available_params.index('xhci') index_y = available_params.index('yhci') for dkey in dkeys: dval = data[dkey] values_x.append(dval[index_x]) values_y.append(dval[index_y]) nc_x[:] = values_x nc_y[:] = values_y log.debug("Writing NetCDF '%s'..." % nc_filename) except Exception as e: log.error("Failed to generate NetCDF '%s'." % nc_filename) raise e finally: nc_handle.close() if not isfile(nc_path): abort(500, "No NetCDF to serve. Looked at '%s'." % nc_path) return send_from_directory(get_path("../cache/"), nc_filename) # API ######################################################################### @app.route("/cache/clear") def cache_clear(): """ Removes all files from the cache that are older than roughly one month. """ a_month_ago = datetime.datetime.now() - datetime.timedelta(days=32) cache_dir = get_path('../cache') removed_files = remove_files_created_before(a_month_ago, cache_dir) return "Cache cleared! Removed %d old file(s)." % len(removed_files) # DEV TOOLS ################################################################### # @app.route("/inspect") # def analyze_cdf(): # """ # For debug purposes. # """ # cdf_to_inspect = get_path("../res/dummy.nc") # cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc") # # si = StringIO.StringIO() # cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length']) # cw.writeheader() # # # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag # cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4") # for variable in cdf_handle.variables: # v = cdf_handle.variables[variable] # cw.writerow({ # 'Name': variable, # 'Shape': v.shape, # 'Length': v.size, # }) # cdf_handle.close() # # return si.getvalue() # MAIN ######################################################################## if __name__ == "__main__": # Debug mode on, as the production server does not use this. extra_files = [get_path('../config.yml')] app.run(debug=True, extra_files=extra_files)