import StringIO
import datetime
import time
import gzip
import json
import logging
import random
import tarfile
import urllib
from csv import writer as csv_writer
from math import sqrt
from os import environ, remove as removefile
from os.path import isfile, join, abspath, dirname
from flask import Flask
from flask import request
from flask import url_for, send_from_directory, abort as abort_flask
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset, date2num
from yaml import load as yaml_load
# PATH RELATIVITY #############################################################
THIS_DIRECTORY = dirname(abspath(__file__))
def get_path(relative_path):
"""Get an absolute path from the relative path to this script directory."""
return abspath(join(THIS_DIRECTORY, relative_path))
# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################
# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
version = version_file.read().strip()
# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
config = yaml_load(config_file.read())
FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"
# LOGGING #####################################################################
log = logging.getLogger("HelioPropa")
log.setLevel(logging.DEBUG)
# log.setLevel(logging.ERROR) # <-- set log level here !
logHandler = logging.FileHandler(get_path('run.log'))
logHandler.setFormatter(logging.Formatter(
"%(asctime)s - %(levelname)s - %(message)s"
))
log.addHandler(logHandler)
# SETUP FLASK ENGINE ##########################################################
app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'
if app.debug:
log.info("Starting Flask app IN DEBUG MODE...")
else:
log.info("Starting Flask app...")
# SETUP JINJA2 TEMPLATE ENGINE ################################################
def static_global(filename):
return url_for('static', filename=filename)
def shuffle_filter(seq):
"""
This shuffles the sequence it is applied to.
Jinja2 _should_ provide this.
"""
try:
result = list(seq)
random.shuffle(result)
return result
except:
return seq
def markdown_filter(value, nl2br=False, p=True):
"""
Converts markdown into html.
nl2br: set to True to replace line breaks with
tags
p: set to False to remove the enclosing
", "").replace(r"
", "") return markdowned tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]), trim_blocks=True, lstrip_blocks=True) tpl_engine.globals.update( url_for=url_for, static=static_global, ) tpl_engine.filters['markdown'] = markdown_filter tpl_engine.filters['md'] = markdown_filter tpl_engine.filters['shuffle'] = shuffle_filter tpl_global_vars = { 'request': request, 'version': version, 'config': config, 'now': datetime.datetime.now(), } # HELPERS ##################################################################### def abort(code, message): log.error(message) abort_flask(code, message) def render_view(view, context=None): """ A simple helper to render [view] template with [context] vars. It automatically adds the global template vars defined above, too. It returns a string, usually the HTML contents to display. """ context = {} if context is None else context return tpl_engine.get_template(view).render( dict(tpl_global_vars.items() + context.items()) ) # def render_page(page, title="My Page", context=None): # """ # A simple helper to render the md_page.html template with [context] vars & # the additional contents of `page/[page].md` in the `md_page` variable. # It automagically adds the global template vars defined above, too. # It returns a string, usually the HTML contents to display. # """ # if context is None: # context = {} # context['title'] = title # context['md_page'] = '' # with file(get_path('page/%s.md' % page)) as f: # context['md_page'] = f.read() # return tpl_engine.get_template('md_page.html').render( # dict(tpl_global_vars.items() + context.items()) # ) def is_list_in_list(needle, haystack): for n in needle: if n not in haystack: return False return True def datetime_from_list(time_list): """ Datetimes in retrieved CDFs are stored as lists of numbers, with DayOfYear starting at 0. We want it starting at 1 for default parsers. """ # Day Of Year starts at 0, but for our datetime parser it starts at 1 doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1) return datetime.datetime.strptime( "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])), "%Y%j%H%M%S%f" ) def get_target_config(slug): for s in config['targets']: # dumb if s['slug'] == slug: return s raise Exception("No target found in configuration for '%s'." % slug) def check_target_config(slug): get_target_config(slug) def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): """ Handles remote querying Myriam's API, downloading, extracting and caching the netCDF files. :param orbiter: key of the source in the YAML config :param what: either 'model' or 'orbit', a key in the config of the source :param started_at: :param stopped_at: :return: a list of local file paths to netCDF (.nc) files """ url = config['amda'].format( dataSet=what, startTime=started_at.isoformat(), stopTime=stopped_at.isoformat() ) log.info("Fetching remote gzip files list at '%s'." % url) retries = 0 success = False errors = [] remote_gzip_files = [] while not success and retries < 3: try: response = urllib.urlopen(url) remote_gzip_files = json.loads(response.read()) if not remote_gzip_files: raise Exception("Failed to fetch data at '%s'." % url) if remote_gzip_files == 'NODATASET': raise Exception("API says there's no dataset at '%s'." % url) if remote_gzip_files == 'ERROR': raise Exception("API returned an error at '%s'." % url) if remote_gzip_files == ['OUTOFTIME']: # it happens raise Exception("API says it's out of time at '%s'." % url) success = True except Exception as e: log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message)) remote_gzip_files = [] errors.append(e) finally: retries += 1 if not remote_gzip_files: abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" % (orbiter, url, errors)) else: remote_gzip_files = list(set(remote_gzip_files)) log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files)) local_gzip_files = [] for remote_gzip_file in remote_gzip_files: # hotfixes to remove when fixed upstream @Myriam if remote_gzip_file in ['OUTOFTIME', 'ERROR']: continue # sometimes half the response is okay, the other not if remote_gzip_file.endswith('/.gz'): continue remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1) ################################################ filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1]) local_gzip_file = get_path("../cache/%s" % filename) local_gzip_files.append(local_gzip_file) if not isfile(local_gzip_file): log.debug("Retrieving '%s'..." % local_gzip_file) urllib.urlretrieve(remote_gzip_file, local_gzip_file) log.debug("Retrieved '%s'." % local_gzip_file) local_netc_files = [] for local_gzip_file in local_gzip_files: local_netc_file = local_gzip_file[0:-3] local_netc_files.append(local_netc_file) log.debug("Unzipping '%s'..." % local_gzip_file) success = True try: with gzip.open(local_gzip_file, 'rb') as f: file_content = f.read() with open(local_netc_file, 'w+b') as g: g.write(file_content) except Exception as e: success = False log.warning("Cannot process gz file '%s' from '%s' : %s" % (local_gzip_file, url, e)) if success: log.debug("Unzipped '%s'." % local_gzip_file) return local_netc_files # The available parameters in the generated CSV and NetCDF files. # The order matters. If you change this you also need to change the # innermost loop of `get_data_for_target`. # The javascript knows the targets' properties under these names. # DEPRECATED, use PARAMETERS instead PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens', 'angl', 'xhci', 'yhci') PARAMETERS = { 'pdyn': { 'slug': 'pdyn', 'name': 'Dyn. Pressure', 'unit': 'nPa', }, 'vlen': { 'slug': 'vlen', 'name': 'Velocity', 'unit': 'km/s', }, 'magn': { 'slug': 'magn', 'name': 'Magnetism', 'unit': 'nT', }, 'temp': { 'slug': 'temp', 'name': 'Temperature', 'unit': 'K', }, 'dens': { 'slug': 'dens', 'name': 'Density', 'unit': 'cm^-3', }, 'angl': { 'slug': 'angl', 'name': 'Angle T-S-E', 'unit': 'deg', }, } def get_data_for_target(target_config, started_at, stopped_at): """ :return: dict whose keys are datetime as str, values tuples of data """ log.debug("Grabbing data for '%s'..." % target_config['slug']) # @todo iterate on models when there are many try: model_slug = target_config['models'][0]['slug'] except Exception as e: abort(500, "Invalid model configuration for '%s' : %s" % (target_config['slug'], str(e))) try: orbit_slug = target_config['orbit']['model'] except Exception as e: abort(500, "Invalid orbit configuration for '%s' : %s" % (target_config['slug'], str(e))) # Grab the list of netCDF files from Myriam's API # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz model_files = retrieve_amda_netcdf(target_config['slug'], model_slug, started_at, stopped_at) orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug, started_at, stopped_at) precision = "%Y-%m-%dT%H" # model and orbits times are equal-ish orbit_data = {} # keys are datetime as str, values arrays of XY for orbit_file in orbit_files: log.debug("%s: opening orbit NETCDF4 '%s'..." % (target_config['name'], orbit_file)) cdf_handle = Dataset(orbit_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_hci = cdf_handle.variables['HCI'] for time, datum_hci in zip(times, data_hci): dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: dkey = dtime.strftime(precision) orbit_data[dkey] = datum_hci cdf_handle.close() all_data = {} # keys are datetime as str, values tuples of data for model_file in model_files: # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn log.debug("%s: opening model NETCDF4 '%s'..." % (target_config['name'], model_file)) cdf_handle = Dataset(model_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_v = cdf_handle.variables['V'] data_b = cdf_handle.variables['B'] data_t = cdf_handle.variables['T'] data_n = cdf_handle.variables['N'] data_p = cdf_handle.variables['P_dyn'] data_d = cdf_handle.variables['Delta_angle'] log.debug("%s: aggregating data from '%s'..." % (target_config['name'], model_file)) for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \ in zip(times, data_v, data_b, data_t, data_n, data_p, data_d): vrad = datum_v[0] vtan = datum_v[1] dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: dkey = dtime.strftime(precision) x_hci = None y_hci = None if dkey in orbit_data: x_hci = orbit_data[dkey][0] y_hci = orbit_data[dkey][1] all_data[dkey] = ( dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"), vrad, vtan, sqrt(vrad * vrad + vtan * vtan), datum_b, datum_t, datum_n, datum_p, datum_d, x_hci, y_hci ) cdf_handle.close() return all_data def generate_csv_contents(target_slug, started_at, stopped_at): target_config = get_target_config(target_slug) log.debug("Crunching CSV contents for '%s'..." % target_config['name']) si = StringIO.StringIO() cw = csv_writer(si) cw.writerow(PROPERTIES) all_data = get_data_for_target(target_config, started_at, stopped_at) log.debug("Writing and sorting CSV for '%s'..." % target_config['slug']) for dkey in sorted(all_data): cw.writerow(all_data[dkey]) log.info("Generated CSV contents for '%s'." % target_config['slug']) return si.getvalue() def generate_csv_file_if_needed(target_slug, started_at, stopped_at): filename = "%s_%s_%s.csv" % (target_slug, started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) if not isfile(local_csv_file): log.info("Generating CSV '%s'..." % local_csv_file) try: with open(local_csv_file, mode="w+") as f: f.write(generate_csv_contents(target_slug, started_at=started_at, stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) except Exception as e: if isfile(local_csv_file): log.warn("Removing failed CSV '%s'..." % local_csv_file) removefile(local_csv_file) abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) def remove_files_created_before(date, in_directory): """ Will throw if something horrible happens. Does not remove recursively (could be done with os.walk if needed). Does not remove directories either. :param date: datetime object :param in_directory: absolute path to directory :return: """ import os import time secs = time.mktime(date.timetuple()) if not os.path.isdir(in_directory): raise ValueError("No directory to clean at '%s'.") removed_files = [] for file_name in os.listdir(in_directory): file_path = os.path.join(in_directory, file_name) if os.path.isfile(file_path): t = os.stat(file_path) if t.st_ctime < secs: os.remove(file_path) removed_files.append(file_path) return removed_files def get_hit_counter(): hit_count_path = get_path("../VISITS") if isfile(hit_count_path): hit_count = int(open(hit_count_path).read()) else: hit_count = 1 return hit_count def increment_hit_counter(): hit_count_path = get_path("../VISITS") if isfile(hit_count_path): hit_count = int(open(hit_count_path).read()) hit_count += 1 else: hit_count = 1 hit_counter_file = open(hit_count_path, 'w') hit_counter_file.write(str(hit_count)) hit_counter_file.close() return hit_count tpl_global_vars['visits'] = get_hit_counter() # ROUTING ##################################################################### @app.route('/favicon.ico') def favicon(): return send_from_directory( join(app.root_path, 'static', 'img'), 'favicon.ico', mimetype='image/vnd.microsoft.icon' ) @app.route("/") @app.route("/home.html") @app.route("/index.html") def home(): increment_hit_counter() return render_view('home.html.jinja2', { 'targets': config['targets'], 'planets': [s for s in config['targets'] if s['type'] == 'planet'], 'probes': [s for s in config['targets'] if s['type'] == 'probe'], 'comets': [s for s in config['targets'] if s['type'] == 'comet'], 'visits': get_hit_counter(), }) @app.route("/